diff options
| author | kraibse <33869366+kraibse@users.noreply.github.com> | 2026-11-02 16:01:57 +0100 |
|---|---|---|
| committer | kraibse <33869366+kraibse@users.noreply.github.com> | 2026-11-02 16:01:57 +0100 |
| commit | b374709aa202d586dd09f81f3389da8d73fea0d7 (patch) | |
| tree | 5c4033d827c517aca9cc73dc497fbb3e81291bf1 | |
| parent | 021c9724371e45fa694c9ee3106b9011364d2853 (diff) | |
| parent | ce1b1de6f1beed36958368f89cda466937ff9716 (diff) | |
Merge remote-tracking branch 'origin/nx2' into codex/upstream-loading-hotkeys
208 files changed, 11108 insertions, 969 deletions
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 20aa57f7aa..2a4e092f75 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -42,8 +42,9 @@ jobs: runs-on: ${{ matrix.runner }} env: CCACHE_DIR: ${{ github.workspace }}/.ccache - CCACHE_MAXSIZE: 1500M + CCACHE_MAXSIZE: 3000M SCCACHE_DIR: ${{ github.workspace }}/.sccache + SCCACHE_CACHE_SIZE: 3G strategy: fail-fast: false matrix: @@ -72,7 +73,7 @@ jobs: steps: - name: Check out source - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Install Linux dependencies if: startsWith(matrix.platform, 'linux') @@ -87,7 +88,9 @@ jobs: libswscale-dev pkg-config zlib1g-dev libva-dev libvdpau-dev \ spirv-tools spirv-headers libusb-1.0-0-dev \ libboost-dev libboost-fiber-dev libboost-context-dev \ - libopus-dev libasound2-dev ccache + libopus-dev libasound2-dev ccache \ + libxcb-cursor0 libxcb-icccm4 libxcb-image0 \ + libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-shape0 libxcb-xinerama0 libxcb-xkb1 # The packages docs/Deps.md lists for macOS, plus ccache. - name: Install macOS dependencies @@ -196,6 +199,7 @@ jobs: with: msystem: UCRT64 update: true + cache: true install: >- git make autoconf libtool automake-wrapper patch jq pactoys ninja pacboy: >- @@ -207,7 +211,7 @@ jobs: lerc:p graphite2:p libwebp:p libtiff:p brotli:p ccache:p - name: Cache CPM downloads - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: .cache/cpm key: cpm-${{ runner.os }}-${{ hashFiles('cpmfile.json', 'CMakeLists.txt', 'src/**/CMakeLists.txt') }} @@ -217,7 +221,7 @@ jobs: # The sha in the key makes every run save a fresh cache (actions/cache never saves over an # exact hit), and restore-keys seeds it from the newest one of the same platform. - name: Cache compiler objects - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | .ccache @@ -369,9 +373,6 @@ jobs: set -euo pipefail test -x build/release/bin/eglantine test -x build/release/bin/eden-cli - # Libraries Qt's xcb platform plugin needs, so linuxdeploy can resolve and bundle them. - sudo apt-get install -y --no-install-recommends libxcb-cursor0 libxcb-icccm4 libxcb-image0 \ - libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-shape0 libxcb-xinerama0 libxcb-xkb1 tools="$RUNNER_TEMP/linuxdeploy" mkdir -p "$tools" arch=$(uname -m) # aarch64 or x86_64 @@ -496,12 +497,19 @@ jobs: codesign --verify --deep --strict "$app" ditto -c -k --keepParent "$app" "eglantine-${GITHUB_SHA}-${{ matrix.platform }}.zip" ls -la "eglantine-${GITHUB_SHA}-${{ matrix.platform }}.zip" + # Also ship a .dmg (drag eglantine.app onto Applications). + dmgdir=$(mktemp -d) + ditto "$app" "$dmgdir/eglantine.app" + ln -s /Applications "$dmgdir/Applications" + hdiutil create -volname "Eglantine" -srcfolder "$dmgdir" -ov -format UDZO \ + "eglantine-${GITHUB_SHA}-${{ matrix.platform }}.dmg" + ls -la "eglantine-${GITHUB_SHA}-${{ matrix.platform }}.dmg" # The artifact is the bin folder itself: GitHub zips artifacts on download, and uploading a # zip gave a zip inside a zip. The release job zips it once for the release. - name: Upload Windows binary if: matrix.platform == 'windows-x86_64' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: eglantine-${{ matrix.platform }}-${{ github.sha }} path: build/release/bin/ @@ -510,7 +518,7 @@ jobs: - name: Upload MinGW binary if: matrix.mingw - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: eglantine-${{ matrix.platform }}-${{ github.sha }} path: build/release/bin/ @@ -520,7 +528,7 @@ jobs: # Both Linux files go in one artifact: the tar.gz and the AppImage. - name: Upload Linux binary if: startsWith(matrix.platform, 'linux') - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: eglantine-${{ matrix.platform }}-${{ github.sha }} path: | @@ -533,10 +541,12 @@ jobs: # Zipped with ditto beforehand: the bundle's framework symlinks don't survive artifact zipping. - name: Upload macOS app if: startsWith(matrix.platform, 'macos') - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: eglantine-${{ matrix.platform }}-${{ github.sha }} - path: eglantine-${{ github.sha }}-${{ matrix.platform }}.zip + path: | + eglantine-${{ github.sha }}-${{ matrix.platform }}.zip + eglantine-${{ github.sha }}-${{ matrix.platform }}.dmg if-no-files-found: error compression-level: 0 retention-days: 7 @@ -553,7 +563,7 @@ jobs: contents: write steps: - name: Download binaries - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v7 with: pattern: eglantine-*-${{ github.sha }} path: artifacts @@ -570,6 +580,7 @@ jobs: cp "$artifact"/*.AppImage "$artifact"/*.tar.gz dist/ elif compgen -G "$artifact*.zip" > /dev/null; then cp "$artifact"/*.zip dist/ + cp "$artifact"/*.dmg dist/ 2>/dev/null || true else (cd "$artifact" && zip -qr "${GITHUB_WORKSPACE}/dist/${name}.zip" .) fi diff --git a/CMakeLists.txt b/CMakeLists.txt index ba44aa9d47..04d0f17d74 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -463,6 +463,9 @@ if (lz4_ADDED) endif() # zstd +# Always bundled: NUP archives are only byte-identical across platforms when every build +# compresses with the same zstd version. +set(zstd_FORCE_BUNDLED ON) AddJsonPackage(zstd) if (zstd_ADDED) diff --git a/NX2_NOTES.md b/NX2_NOTES.md index 08ce0123a1..eeebb90568 100644 --- a/NX2_NOTES.md +++ b/NX2_NOTES.md @@ -144,7 +144,7 @@ With the patched decoder, games 1/3/4 only leave ~60-110 rare SIMD encodings in | 86 | title 1 after its title screen: black frames | tens of thousands of rejected shaders: IPA interpolation modes, integer carry/third source, uniform comparison, formatted surface load/store, uniform constant load, unsupported opcodes | open | | 87 | title 1 loading crash: Vulkan out of memory | an 8192x8192x68-layer virtual texture (~23 GB) | images over 2^28 texels are created sparse with no memory (`Unbacked`: no uploads/downloads) (96a9b81); the game reaches its real title screen ("Press L + R"), ground black (that texture has no data) | | 88 | title 1 shader coverage | checked against nvdisasm (`nx2work/nvd.py` disassembles one instruction at a time, since a single illegal word stops nvdisasm); calls: RET may restore the link through IMAD.MOV with code between, library functions may EXIT, `@Pg EXIT Pc`; PLOP3/UPLOP3 any table, UP2UR, ULOP3 predicate output; UIMAD.WIDE, uniform USHF/FMNMX/I2F/F2I, LDS `[Ra.Xn+URb]`, ATOMG/RED/STG forms, PRMT, FSWZADD, REDUX.MAX | 1351c73, 52cda01, 49396b7: 1699 of 1712 dumped programs lower; left: 12 task/mesh programs (ISBERD/ISBEWR, row 89) and one SURED | -| 89 | task/mesh shaders | title 1 draws some geometry with NV-style task + mesh programs: the task program culls a batch and writes its payload with `ISBEWR.O.ATTR [Rn+0x10]` and the mesh count with `ISBEWR [0x4]`; mesh programs read it back with `ISBERD.ATTR.SKEW`. The most-drawn rejected program (0x114c459500) is a task program. Eden has no mesh pipeline | open: needs a task/mesh pipeline (VK_EXT_mesh_shader) or a compute + indirect-draw emulation | +| 89 | task/mesh shaders | title 1 draws some geometry with NVN task + mesh programs. While SET_MESH_CONTROL (0x114c) is on: task program in the vertex slot, mesh program in the tessellation slot, header-only pass-through geometry program (its output map = per-primitive outputs); draw count = task workgroups. Task: `ISBERD.ATTR.SKEW [RZ]` = workgroup index, payload `ISBEWR.O.ATTR [rank*stride+0x10]`, mesh count `ISBEWR.O.ATTR [0x4]`. Mesh: input [0] = its index, payload from 0x10; vertices in groups of 32, one 128-byte line per output component (mesh header omap order), then the per-primitive groups; map: count `ISBEWR.O.32 [0x3]`, U8 indices from 4. Addresses are plain linear bytes | task/mesh stages for VK_EXT_mesh_shader: ISBE ops kept as a pseudo instruction through the lowering, emulated stage buffer (task payload / workgroup memory), every EXIT to one epilogue that emits EmitMeshTasks or SetMeshOutputs + the output copy; pipelines keyed by SET_MESH_SHADER_A/B and SET_MESH_INIT_SHADER, kept out of the disk cache. All captured programs translate to valid SPIR-V; in the attract demo they draw cleanly at ~3 ms a frame, and without them the demo stalls on the world map (`EDEN_NX2_NO_MESH`, `EDEN_NX2_NO_MESH_FILE` to compare) | | 90 | each rejected program was lowered and logged again on every draw (1.7M log lines in 13 min) | no negative cache | the shader cache keeps the first exception per address until that memory is written; logs deduplicated (bcb4ff0) | | 91 | a compute program with a jump table: "Failed to track indirect branch" | Eden's tracker wants IMNMX imm -> SHL imm -> LDC -> BRX; IMAD.SHL was lowered to an XMAD multiply | IMAD by a power of two lowers to SHL (49396b7+) | | 92 | with rows 88's compute forms, title 1 stays on its loading map (file reads stop ~46 s in) | two compute programs failing in Eden's compiler: a jump table whose targets lie past the first EXIT, and a 128-bit store of RZ from unaligned scratch registers | programs with BRX end at the compiler's closing branch-to-self; R248-R251 for 128-bit zero stores (3df6a81) | @@ -152,7 +152,7 @@ With the patched decoder, games 1/3/4 only leave ~60-110 rare SIMD encodings in | 94 | near terrain black | 8192x8192x68 ASTC 8x8 virtual textures: 1.5 GB in guest memory, 23 GB each as Eden's RGBA8 decode | huge GPU-decoded ASTC arrays keep levels from the first that fits 512 MiB (here 3) on the host (60c4791) | | 95 | wind/foliage shards | SM70+ MUFU.SIN/COS take turns (sin(2pi x)); Eden's Maxwell MUFU takes radians | scale by 2pi (245e6ad); jungle canopy not re-checked yet | | 96 | title 1 aborts after ~1.5-4 min at the title or when a race starts (nn::diag::Abort from main+0x19e5df0) | the movie player (main+0x19e5860) lays out its decoder frames with nn::codec::CalculateDecoderImagePoolBuffer, which sends nx2:gsv 991; auto-stubbed, the reply kept the request words, so the frame size read as 0x167027f'00000001 and the budget check at main+0x19e5b58 failed. (Not the "video-frame-count" < 9 check: the game sets that from a two-slot registry, at most 2.) | gsv 991 answers a block-linear NV12 size (640x360 -> 0x64000) | -| 97 | pop-in, objects flicker in and out, low-quality meshes | high-detail LODs drawn through task/mesh programs (bound in stage slot 1, header type VS, ISBERD/ISBEWR), which are rejected and skipped | open: task/mesh support | +| 97 | pop-in, objects flicker in and out, low-quality meshes | high-detail LODs drawn through task/mesh programs (bound in stage slot 1, header type VS, ISBERD/ISBEWR), which are rejected and skipped | task/mesh support (row 89); not yet checked in a race | | 98 | ~12 fps at the title | host GPU ~20% busy, emulated cores ~0.9 host cores, GPU thread 45-70% busy, much of it waiting on the host GPU: semaphore acquires wait for host fences under GPU accuracy High; ASTC uploads end in Finish() (a Flush() there crashed at boot); a 1 ms poll sleep while channels are suspended | gpu_accuracy=Low: ~16 fps; SDL main loop capped at 240 Hz (88f04ad). Open: ASTC upload without a full stall (and per-layer re-upload of the trimmed virtual textures), native MME interpreter cost, draw path | | 99 | VRAM over the card (8.4-9.8 GB used on an 8 GB RTX 4070 Laptop), texture cache always collecting | ~700 ASTC 8x8 textures decoded to RGBA8 (16x their size, ~4.5 GB) | astc_recompression=Bc3 + accelerate_astc=CpuAsynchronous in the config; huge ASTC arrays stay on the GPU decoder as trimmed RGBA8 (ed0a3c9): 4.6-5.6 GB, ~10 -> 16 fps | | 100 | 8 ImageTranscode threads at ~44% each (3.5 cores) | Eden's default pool is half the hardware threads | an eighth, at least 2 (`EDEN_NX2_TRANSCODE_THREADS` overrides): ~19 fps on the attract loop | diff --git a/src/audio_core/audio_render_manager.cpp b/src/audio_core/audio_render_manager.cpp index 8c4e74c4be..4034de1ec3 100644 --- a/src/audio_core/audio_render_manager.cpp +++ b/src/audio_core/audio_render_manager.cpp @@ -33,6 +33,8 @@ SystemManager& Manager::GetSystemManager() { Result Manager::GetWorkBufferSize(const AudioRendererParameterInternal& params, u64& out_count) const { if (!CheckValidRevision(params.revision)) { + LOG_ERROR(Service_Audio, "Audio renderer revision {:#x} ({}) is not supported", + params.revision, GetRevisionNum(params.revision)); return Service::Audio::ResultInvalidRevision; } diff --git a/src/audio_core/common/feature_support.h b/src/audio_core/common/feature_support.h index 4585be45ef..d6571c0661 100644 --- a/src/audio_core/common/feature_support.h +++ b/src/audio_core/common/feature_support.h @@ -17,6 +17,16 @@ namespace AudioCore { constexpr u32 CurrentRevision = 15; +constexpr u32 NativeCurrentRevision = 16; + +constexpr bool IsNativeRevision(u32 revision) { + return (revision & 0xffffff) == Common::MakeMagic('R', 'E', 'v', '\0'); +} + +constexpr u32 EncodeRevision(u32 revision, bool native) { + return Common::MakeMagic('R', 'E', native ? 'v' : 'V', + static_cast<char>('0' + revision)); +} enum class SupportTags { CommandProcessingTimeEstimatorVersion4, @@ -57,12 +67,15 @@ enum class SupportTags { }; constexpr u32 GetRevisionNum(u32 user_revision) { - if (user_revision >= 0x100) { - user_revision -= Common::MakeMagic('R', 'E', 'V', '0'); - user_revision >>= 24; + if (user_revision < 0x100) { + return user_revision; } - - return user_revision; + if (!IsNativeRevision(user_revision) && + (user_revision & 0xffffff) != Common::MakeMagic('R', 'E', 'V', '\0')) { + return ~u32{0}; + } + const u32 encoded = user_revision >> 24; + return encoded >= '0' ? encoded - '0' : ~u32{0}; }; constexpr bool CheckFeatureSupported(SupportTags tag, u32 user_revision) { @@ -108,11 +121,12 @@ constexpr bool CheckFeatureSupported(SupportTags tag, u32 user_revision) { return false; } user_revision = GetRevisionNum(user_revision); - return (*feature).second <= user_revision; + return user_revision != ~u32{0} && (*feature).second <= user_revision; } constexpr bool CheckValidRevision(u32 user_revision) { - return GetRevisionNum(user_revision) <= CurrentRevision; + return GetRevisionNum(user_revision) <= + (IsNativeRevision(user_revision) ? NativeCurrentRevision : CurrentRevision); }; } // namespace AudioCore diff --git a/src/audio_core/renderer/behavior/behavior_info.cpp b/src/audio_core/renderer/behavior/behavior_info.cpp index 63b80c503a..983cfd5b85 100644 --- a/src/audio_core/renderer/behavior/behavior_info.cpp +++ b/src/audio_core/renderer/behavior/behavior_info.cpp @@ -16,8 +16,7 @@ u32 BehaviorInfo::GetProcessRevisionNum() const { } u32 BehaviorInfo::GetProcessRevision() const { - return Common::MakeMagic('R', 'E', 'V', - static_cast<char>(static_cast<u8>('0') + process_revision)); + return EncodeRevision(process_revision, native_platform); } u32 BehaviorInfo::GetUserRevisionNum() const { @@ -25,11 +24,12 @@ u32 BehaviorInfo::GetUserRevisionNum() const { } u32 BehaviorInfo::GetUserRevision() const { - return Common::MakeMagic('R', 'E', 'V', - static_cast<char>(static_cast<u8>('0') + user_revision)); + return EncodeRevision(user_revision, native_platform); } void BehaviorInfo::SetUserLibRevision(const u32 user_revision_) { + native_platform = IsNativeRevision(user_revision_); + process_revision = native_platform ? NativeCurrentRevision : CurrentRevision; user_revision = GetRevisionNum(user_revision_); } diff --git a/src/audio_core/renderer/behavior/behavior_info.h b/src/audio_core/renderer/behavior/behavior_info.h index 757f163c5a..318d4e1ff4 100644 --- a/src/audio_core/renderer/behavior/behavior_info.h +++ b/src/audio_core/renderer/behavior/behavior_info.h @@ -400,6 +400,7 @@ public: u32 process_revision; /// User version u32 user_revision{}; + bool native_platform{}; /// Behaviour flags Flags flags{}; /// Errors generated and reported during Update diff --git a/src/audio_core/renderer/behavior/info_updater.cpp b/src/audio_core/renderer/behavior/info_updater.cpp index ca3eaece94..a8fbcf9730 100644 --- a/src/audio_core/renderer/behavior/info_updater.cpp +++ b/src/audio_core/renderer/behavior/info_updater.cpp @@ -122,12 +122,21 @@ Result InfoUpdater::UpdateVoices(VoiceContext& voice_context, u8 flush_wave_buffer_count; u16 reserved2; VoiceInfo::Flags flags; + u8 reserved_after_flags; SrcQuality src_quality; + u8 crossfade_curve; u32 external_ctx; u32 external_ctx_size; u32 reserved3[2]; }; + static_assert(sizeof(VoiceInParameterV2) == 0x188); + static_assert(offsetof(VoiceInParameterV2, crossfade_curve) == 0x177); const auto* vin = reinterpret_cast<const VoiceInParameterV2*>(input + i * in_stride); + if (behaviour.GetUserRevisionNum() >= 16 && vin->in_use && vin->crossfade_curve != 0) { + LOG_ERROR(Service_Audio, "Unsupported voice biquad crossfade curve {}", + vin->crossfade_curve); + return Service::Audio::ResultInvalidUpdateInfo; + } local_in.id = vin->id; local_in.node_id = vin->node_id; local_in.is_new = vin->is_new; diff --git a/src/audio_core/renderer/sink/circular_buffer_sink_info.cpp b/src/audio_core/renderer/sink/circular_buffer_sink_info.cpp index 0ede02b6b4..7e25413fe1 100644 --- a/src/audio_core/renderer/sink/circular_buffer_sink_info.cpp +++ b/src/audio_core/renderer/sink/circular_buffer_sink_info.cpp @@ -3,7 +3,6 @@ #include "audio_core/renderer/memory/pool_mapper.h" #include "audio_core/renderer/sink/circular_buffer_sink_info.h" -#include "audio_core/renderer/upsampler/upsampler_manager.h" namespace AudioCore::Renderer { @@ -17,13 +16,9 @@ CircularBufferSinkInfo::CircularBufferSinkInfo() { } void CircularBufferSinkInfo::CleanUp() { - auto state_{reinterpret_cast<DeviceState*>(state.data())}; - - if (state_->upsampler_info) { - state_->upsampler_info->manager->Free(state_->upsampler_info); - state_->upsampler_info = nullptr; - } - + // Only device sinks own an upsampler. This sink's state is a CircularBufferState, whose + // first words are buffer positions: reading them as DeviceState::upsampler_info freed a + // garbage pointer when a game changed or removed its circular buffer sink. parameter.fill(0); type = Type::Invalid; } diff --git a/src/common/fiber.cpp b/src/common/fiber.cpp index 0e7f28c5b9..613ac97e22 100644 --- a/src/common/fiber.cpp +++ b/src/common/fiber.cpp @@ -25,8 +25,10 @@ struct Fiber::FiberImpl { FiberImpl() {} u32 canary_1 = CANARY_VALUE; - std::array<u8, DEFAULT_STACK_SIZE> stack{}; - std::array<u8, DEFAULT_STACK_SIZE> rewind_stack{}; + // Left uninitialized: the host gives a stack's pages memory only as they are used. Zero-filled + // std::arrays (and a second stack nothing switched to) made every fiber, one per guest thread, + // hold 4 MiB: ~750 MiB for title 1's ~190 threads. + std::unique_ptr<u8[]> stack; u32 canary_2 = CANARY_VALUE; boost::context::detail::fcontext_t context{}; @@ -38,7 +40,6 @@ struct Fiber::FiberImpl { std::shared_ptr<Fiber> previous_fiber; u8* stack_limit = nullptr; - u8* rewind_stack_limit = nullptr; bool is_thread_fiber = false; bool released = false; }; @@ -49,10 +50,10 @@ void Fiber::SetRewindPoint(std::function<void()>&& rewind_func) { Fiber::Fiber(std::function<void()>&& entry_point_func) : impl{std::make_unique<FiberImpl>()} { impl->entry_point = std::move(entry_point_func); - impl->stack_limit = impl->stack.data(); - impl->rewind_stack_limit = impl->rewind_stack.data(); + impl->stack.reset(new u8[DEFAULT_STACK_SIZE]); + impl->stack_limit = impl->stack.get(); u8* stack_base = impl->stack_limit + DEFAULT_STACK_SIZE; - impl->context = boost::context::detail::make_fcontext(stack_base, impl->stack.size(), [](boost::context::detail::transfer_t transfer) -> void { + impl->context = boost::context::detail::make_fcontext(stack_base, DEFAULT_STACK_SIZE, [](boost::context::detail::transfer_t transfer) -> void { auto* fiber = static_cast<Fiber*>(transfer.data); ASSERT(fiber && fiber->impl && fiber->impl->previous_fiber && fiber->impl->previous_fiber->impl); ASSERT(fiber->impl->canary_1 == CANARY_VALUE); diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 75a9da80e9..2ef1982cbc 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -51,11 +51,13 @@ #endif // ^^^ POSIX ^^^ +#include <cstdlib> #include <mutex> #include <random> #include "common/alignment.h" #include "common/assert.h" +#include "common/div_ceil.h" #include "common/free_region_manager.h" #include "common/host_memory.h" #include "common/logging.h" @@ -517,6 +519,14 @@ public: , virtual_size{virtual_size_} {} +#ifdef __linux__ + bool PunchHole(size_t offset, size_t length) { + return fd >= 0 && + fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, static_cast<off_t>(offset), + static_cast<off_t>(length)) == 0; + } +#endif + bool Init() { long page_size = sysconf(_SC_PAGESIZE); ASSERT_MSG(page_size == 0x1000, "page size {:#x} is incompatible with 4K paging", page_size); @@ -707,7 +717,9 @@ private: HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) : backing_size(backing_size_) , virtual_size(virtual_size_) + , num_granules(Common::DivCeil(backing_size_, size_t{1} << GranuleShift)) { + granules = std::make_unique<std::atomic<u8>[]>(num_granules); #if defined(__OPENORBIS__) || defined(__managarm__) LOG_WARNING(HW_Memory, "Platform doesn't support fastmem"); backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); @@ -790,8 +802,98 @@ void HostMemory::Protect(size_t virtual_offset, size_t length, MemoryPermission #endif } +void HostMemory::HintCold(size_t physical_offset, size_t length, + std::optional<size_t> virtual_offset) const { + if (length == 0 || backing_base == nullptr) { + return; + } + const auto hint = [length](u8* address) { +#ifdef _WIN32 + // Unlocking pages that are not locked takes them out of the working set. + VirtualUnlock(address, length); +#elif defined(__linux__) && defined(MADV_PAGEOUT) + madvise(address, length, MADV_PAGEOUT); +#else + static_cast<void>(address); +#endif + }; + hint(backing_base + physical_offset); + if (virtual_offset && virtual_base != nullptr && *virtual_offset + length <= virtual_size) { + hint(virtual_base + *virtual_offset); + } +} + void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value) { - std::memset(backing_base + physical_offset, fill_value, length); + if (length == 0) { + return; + } + MarkUsed(physical_offset, length); + if (fill_value != 0) { + std::memset(backing_base + physical_offset, fill_value, length); + MarkDirty(physical_offset, length); + return; + } + // Fresh backing memory is zero and the host gives it no RAM until it's touched: clear only + // the granules that may hold data (title 1's heap made the host commit every page of it). + // A granule cleared in full is zero again. +#ifdef __linux__ + // Handing a whole granule back to the kernel frees its RAM; a memset would commit it. + static const bool punch = [] { + const char* value = std::getenv("EDEN_PUNCH_HOLE"); + return value == nullptr || value[0] != '0'; + }(); +#endif + const size_t end = physical_offset + length; + for (size_t offset = physical_offset; offset < end;) { + const size_t index = offset >> GranuleShift; + const size_t granule_begin = index << GranuleShift; + const size_t granule_end = granule_begin + (size_t{1} << GranuleShift); + const size_t clear_end = (std::min)(granule_end, end); + if ((granules[index].load(std::memory_order_acquire) & GranuleDirty) != 0) { + const bool whole = offset == granule_begin && clear_end == granule_end; + bool cleared = false; +#ifdef __linux__ + cleared = whole && punch && impl && impl->PunchHole(granule_begin, granule_end - granule_begin); +#endif + if (!cleared) { + std::memset(backing_base + offset, 0, clear_end - offset); + } + if (whole) { + granules[index].fetch_and(static_cast<u8>(~GranuleDirty), std::memory_order_release); + } + } + offset = clear_end; + } +} + +void HostMemory::MarkUsed(size_t physical_offset, size_t length) { + if (length == 0) { + return; + } + const size_t last = (physical_offset + length - 1) >> GranuleShift; + for (size_t index = physical_offset >> GranuleShift; index <= last; ++index) { + if ((granules[index].load(std::memory_order_relaxed) & GranuleUsed) == 0) { + granules[index].fetch_or(GranuleUsed, std::memory_order_release); + } + } +} + +void HostMemory::MarkDirty(size_t physical_offset, size_t length) { + if (length == 0) { + return; + } + const size_t last = (physical_offset + length - 1) >> GranuleShift; + for (size_t index = physical_offset >> GranuleShift; index <= last; ++index) { + granules[index].fetch_or(GranuleDirty, std::memory_order_release); + } +} + +void HostMemory::ForgetContents() { + for (size_t index = 0; index < num_granules; ++index) { + if ((granules[index].load(std::memory_order_relaxed) & GranuleUsed) != 0) { + granules[index].fetch_or(GranuleDirty, std::memory_order_release); + } + } } void HostMemory::EnableDirectMappedAddress() { diff --git a/src/common/host_memory.h b/src/common/host_memory.h index 8747a1850f..84315091c5 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -6,6 +6,7 @@ #pragma once +#include <atomic> #include <memory> #include <optional> #include "common/common_funcs.h" @@ -52,8 +53,24 @@ public: void EnableDirectMappedAddress(); + /// Clears guest memory being handed out. Memory still zero since it was created (or since its + /// last full clear) is skipped: clearing it only made the host give every page real memory. void ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value); + /// Memory written once and read again rarely: its pages leave the working set (backing and, + /// when given, its view at virtual_offset), so the OS compresses or pages them out first when + /// it needs RAM. The contents stay; using the pages brings them back. + void HintCold(size_t physical_offset, size_t length, + std::optional<size_t> virtual_offset) const; + /// Pages the kernel hands out (cleared or not). + void MarkUsed(size_t physical_offset, size_t length); + + /// Pages given back to the kernel: their contents are no longer known to be zero. + void MarkDirty(size_t physical_offset, size_t length); + + /// A new kernel on this memory: everything a previous one used may hold data. + void ForgetContents(); + [[nodiscard]] u8* BackingBasePointer() noexcept { return backing_base; } @@ -88,6 +105,12 @@ private: u8* backing_base{}; u8* virtual_base{}; size_t virtual_base_offset{}; + + static constexpr size_t GranuleShift = 16; ///< 64 KiB of backing per state byte + static constexpr u8 GranuleUsed = 1 << 0; ///< Handed out at least once + static constexpr u8 GranuleDirty = 1 << 1; ///< May hold non-zero data + std::unique_ptr<std::atomic<u8>[]> granules; + size_t num_granules{}; // Windows requires it for kernels whom lack proper support for some functions! bool fallback_buffer{false}; }; diff --git a/src/common/settings.h b/src/common/settings.h index 97103f457d..74e2006d15 100644 --- a/src/common/settings.h +++ b/src/common/settings.h @@ -573,8 +573,10 @@ struct Values { true, true}; + // BC3 by default: GPUs without ASTC otherwise keep ASTC textures as RGBA8, four times the + // VRAM (Mario Kart World: 7.3-7.8 GB of an 8 GB card, the rest paged into system RAM). SwitchableSetting<AstcRecompression, true> astc_recompression{linkage, - AstcRecompression::Uncompressed, + AstcRecompression::Bc3, "astc_recompression", Category::RendererAdvanced}; diff --git a/src/common/sparse_large_vector.cpp b/src/common/sparse_large_vector.cpp index b96340999d..34aba73d78 100644 --- a/src/common/sparse_large_vector.cpp +++ b/src/common/sparse_large_vector.cpp @@ -39,12 +39,12 @@ static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) { std::lock_guard lock(vector_regions_mutex); for (auto region: vector_regions) { auto addr_shifted = exception_addr >> HostPageBits; - if (region.first <= addr_shifted && addr_shifted <= region.second) { + if (region.first <= addr_shifted && addr_shifted < region.second) { addr = addr_shifted; } // Page-boundary accesses - if (auto addr_ = (exception_addr + 0x40) >> HostPageBits; addr_ != addr_shifted && region.first <= addr_ && addr_ <= region.second) { + if (auto addr_ = (exception_addr + 0x40) >> HostPageBits; addr_ != addr_shifted && region.first <= addr_ && addr_ < region.second) { addr2 = addr_; } @@ -164,7 +164,8 @@ void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept { return; #ifdef _WIN32 std::lock_guard lock(vector_regions_mutex); - std::erase_if(vector_regions, [base](const auto& r) {return r.first == reinterpret_cast<u64>(base); }); + // Regions are kept in host pages. + std::erase_if(vector_regions, [base](const auto& r) {return r.first == reinterpret_cast<u64>(base) >> HostPageBits; }); ASSERT(VirtualFree(base, 0, MEM_RELEASE)); #else ASSERT(munmap(base, size) == 0); diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 0075838d49..eeb3202d12 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -132,6 +132,10 @@ add_library(core STATIC file_sys/program_metadata.h file_sys/registered_cache.cpp file_sys/registered_cache.h + file_sys/nup_container.cpp + file_sys/nup_container.h + file_sys/nup_writer.cpp + file_sys/nup_writer.h file_sys/romfs.cpp file_sys/romfs.h file_sys/romfs_factory.cpp @@ -527,6 +531,7 @@ add_library(core STATIC hle/service/audio/audio_out.h hle/service/audio/audio_renderer_manager.cpp hle/service/audio/audio_renderer_manager.h + hle/service/audio/nx2_audio_device_manager.h hle/service/audio/nx2_audio_user_client.h hle/service/audio/audio_renderer.cpp hle/service/audio/audio_renderer.h @@ -1155,6 +1160,8 @@ add_library(core STATIC loader/nso.h loader/nsp.cpp loader/nsp.h + loader/nup.cpp + loader/nup.h loader/xci.cpp loader/xci.h memory.cpp diff --git a/src/core/arm/dynarmic/arm_dynarmic_32.cpp b/src/core/arm/dynarmic/arm_dynarmic_32.cpp index a59aa3cac3..3a875493c6 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_32.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_32.cpp @@ -416,6 +416,10 @@ void ArmDynarmic32::ClearInstructionCache() { } void ArmDynarmic32::InvalidateCacheRange(u64 addr, std::size_t size) { + // Drop the cached code page too: blocks recompiled after the invalidation would + // otherwise still decode the old instructions (e.g. a gdbstub breakpoint written + // into or removed from that page). + m_cb->last_code_addr = u64(-1); m_jit->InvalidateCacheRange(static_cast<u32>(addr), size); } diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp index e65e38dd9e..c5be4d2520 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp @@ -481,6 +481,10 @@ void ArmDynarmic64::ClearInstructionCache() { } void ArmDynarmic64::InvalidateCacheRange(u64 addr, std::size_t size) { + // Drop the cached code page too: blocks recompiled after the invalidation would + // otherwise still decode the old instructions (e.g. a gdbstub breakpoint written + // into or removed from that page). + m_cb->last_code_addr = u64(-1); m_jit->InvalidateCacheRange(addr, size); } diff --git a/src/core/core.cpp b/src/core/core.cpp index 41871df870..3842ee25a8 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -25,6 +25,7 @@ #include "core/debugger/debugger.h" #include "core/device_memory.h" #include "core/file_sys/fs_filesystem.h" +#include "core/file_sys/nup_container.h" #include "core/file_sys/patch_manager.h" #include "core/file_sys/program_metadata.h" #include "core/file_sys/registered_cache.h" @@ -116,9 +117,27 @@ namespace { // Switch 2 titles boot as a loose ExeFS, with main.npdm next to main. bool IsSwitch2Program(const FileSys::VirtualFile& file) { + FileSys::ProgramMetadata metadata; + if (FileSys::NupContainer::IsNupFile(file)) { + FileSys::NupContainer nup{file}; + if (nup.GetStatus() != Loader::ResultStatus::Success) { + return false; + } + std::optional<std::size_t> program; + if (const auto base = nup.FindBasePackage(); + base && nup.GetExeFSDir(*base) != nullptr) { + program = base; + } else if (const auto update = nup.FindBestUpdatePackage(); + update && nup.GetExeFSDir(*update) != nullptr) { + program = update; + } + const auto exefs = program ? nup.GetExeFSDir(*program) : nullptr; + const auto npdm = exefs != nullptr ? exefs->GetFile("main.npdm") : nullptr; + return npdm != nullptr && metadata.Load(npdm) == Loader::ResultStatus::Success && + metadata.IsSwitch2(); + } const auto dir = file != nullptr ? file->GetContainingDirectory() : nullptr; const auto npdm = dir != nullptr ? dir->GetFile("main.npdm") : nullptr; - FileSys::ProgramMetadata metadata; return npdm != nullptr && metadata.Load(npdm) == Loader::ResultStatus::Success && metadata.IsSwitch2(); } diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index 27ff342213..d275bdbf2d 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -18,6 +18,7 @@ #include "common/common_types.h" #include "common/range_mutex.h" #include "common/scratch_buffer.h" +#include "common/thread_worker.h" #include "common/sparse_large_vector.h" namespace Core { @@ -53,6 +54,10 @@ public: void BindInterface(DeviceInterface* device_inter); + /// Marks the memory behind [address, address + size) cold (Common::HostMemory::HintCold), + /// from a low-priority thread: data the host GPU has its own copy of, e.g. uploaded textures. + void HintCold(DAddr address, size_t size); + DAddr Allocate(size_t size); void AllocateFixed(DAddr start, size_t size); void Free(DAddr start, size_t size); @@ -197,7 +202,9 @@ private: std::unique_ptr<DeviceMemoryManagerAllocator<Traits>> impl; const uintptr_t physical_base; + const DeviceMemory* device_memory_ptr; DeviceInterface* device_inter; + std::unique_ptr<Common::ThreadWorker> cold_hint_worker; struct TrackedEntry { VAddr cpu_backing_address; diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 83faa1b3f3..3523e3cf1c 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -171,6 +171,7 @@ struct DeviceMemoryManagerAllocator { template <typename Traits> DeviceMemoryManager<Traits>::DeviceMemoryManager(const DeviceMemory& device_memory_) : physical_base{uintptr_t(device_memory_.buffer.BackingBasePointer())} + , device_memory_ptr{&device_memory_} , device_inter{nullptr} , compressed_device_addr(1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS)) , tracked_entries(device_as_size >> Memory::YUZU_PAGEBITS) @@ -183,6 +184,49 @@ template <typename Traits> DeviceMemoryManager<Traits>::~DeviceMemoryManager() = default; template <typename Traits> +void DeviceMemoryManager<Traits>::HintCold(DAddr address, size_t size) { + struct Run { + u64 physical_offset; + VAddr virtual_address; ///< 0: no CPU mapping known + u64 size; + }; + std::vector<Run> runs; + const size_t page_end = Common::DivCeil(address + size, Memory::YUZU_PAGESIZE); + for (size_t page = address >> Memory::YUZU_PAGEBITS; page < page_end; ++page) { + const u32 compressed = tracked_entries[page].compressed_physical_ptr; + if (compressed == 0) { + continue; + } + const u64 physical = u64{compressed - 1} << Memory::YUZU_PAGEBITS; + const VAddr vaddr = ExtractCPUBacking(page).second; + if (!runs.empty()) { + Run& last = runs.back(); + const bool contiguous_view = + last.virtual_address == 0 ? vaddr == 0 : last.virtual_address + last.size == vaddr; + if (last.physical_offset + last.size == physical && contiguous_view) { + last.size += Memory::YUZU_PAGESIZE; + continue; + } + } + runs.push_back({physical, vaddr, Memory::YUZU_PAGESIZE}); + } + if (runs.empty()) { + return; + } + if (!cold_hint_worker) { + cold_hint_worker = std::make_unique<Common::ThreadWorker>(1, "DeviceMemory:ColdHint"); + } + cold_hint_worker->QueueWork([this, runs = std::move(runs)] { + for (const Run& run : runs) { + device_memory_ptr->buffer.HintCold( + run.physical_offset, run.size, + run.virtual_address != 0 ? std::optional<size_t>{run.virtual_address} + : std::nullopt); + } + }); +} + +template <typename Traits> void DeviceMemoryManager<Traits>::BindInterface(DeviceInterface* device_inter_) { device_inter = device_inter_; } diff --git a/src/core/file_sys/fsmitm_romfsbuild.cpp b/src/core/file_sys/fsmitm_romfsbuild.cpp index 8807bbd0f7..d34e24f177 100644 --- a/src/core/file_sys/fsmitm_romfsbuild.cpp +++ b/src/core/file_sys/fsmitm_romfsbuild.cpp @@ -55,15 +55,18 @@ static_assert(sizeof(RomFSFileEntry) == 0x20, "RomFSFileEntry has incorrect size struct RomFSBuildFileContext; +// The context's directories and files own the nodes; the links between them don't, or every +// parent and child would own each other and no build would ever be freed (title 1's loose RomFS: +// 206k files, ~50 MiB per build). struct RomFSBuildDirectoryContext { std::string path; u32 cur_path_ofs = 0; u32 path_len = 0; u32 entry_offset = 0; - std::shared_ptr<RomFSBuildDirectoryContext> parent; - std::shared_ptr<RomFSBuildDirectoryContext> child; - std::shared_ptr<RomFSBuildDirectoryContext> sibling; - std::shared_ptr<RomFSBuildFileContext> file; + RomFSBuildDirectoryContext* parent = nullptr; + RomFSBuildDirectoryContext* child = nullptr; + RomFSBuildDirectoryContext* sibling = nullptr; + RomFSBuildFileContext* file = nullptr; }; struct RomFSBuildFileContext { @@ -73,8 +76,8 @@ struct RomFSBuildFileContext { u32 entry_offset = 0; u64 offset = 0; u64 size = 0; - std::shared_ptr<RomFSBuildDirectoryContext> parent; - std::shared_ptr<RomFSBuildFileContext> sibling; + RomFSBuildDirectoryContext* parent = nullptr; + RomFSBuildFileContext* sibling = nullptr; VirtualFile source; }; @@ -168,7 +171,7 @@ bool RomFSBuildContext::AddDirectory(std::shared_ptr<RomFSBuildDirectoryContext> num_dirs++; dir_table_size += sizeof(RomFSDirectoryEntry) + Common::AlignUp(dir_ctx->path_len - dir_ctx->cur_path_ofs, 4); - dir_ctx->parent = std::move(parent_dir_ctx); + dir_ctx->parent = parent_dir_ctx.get(); directories.emplace_back(std::move(dir_ctx)); return true; @@ -180,7 +183,7 @@ bool RomFSBuildContext::AddFile(std::shared_ptr<RomFSBuildDirectoryContext> pare num_files++; file_table_size += sizeof(RomFSFileEntry) + Common::AlignUp(file_ctx->path_len - file_ctx->cur_path_ofs, 4); - file_ctx->parent = std::move(parent_dir_ctx); + file_ctx->parent = parent_dir_ctx.get(); files.emplace_back(std::move(file_ctx)); return true; @@ -245,11 +248,11 @@ std::vector<std::pair<u64, VirtualFile>> RomFSBuildContext::Build() { Common::AlignUp(cur_file->path_len - cur_file->cur_path_ofs, 4)); prev_file = cur_file; } - // Assign deferred parent/sibling ownership. + // Assign deferred parent/sibling links. for (auto it = files.rbegin(); it != files.rend(); ++it) { auto& cur_file = *it; cur_file->sibling = cur_file->parent->file; - cur_file->parent->file = cur_file; + cur_file->parent->file = cur_file.get(); } // Determine directory offsets. @@ -260,11 +263,11 @@ std::vector<std::pair<u64, VirtualFile>> RomFSBuildContext::Build() { static_cast<u32>(sizeof(RomFSDirectoryEntry) + Common::AlignUp(cur_dir->path_len - cur_dir->cur_path_ofs, 4)); } - // Assign deferred parent/sibling ownership. + // Assign deferred parent/sibling links. for (auto it = directories.rbegin(); (*it) != root; ++it) { auto& cur_dir = *it; cur_dir->sibling = cur_dir->parent->child; - cur_dir->parent->child = cur_dir; + cur_dir->parent->child = cur_dir.get(); } // Create output map. diff --git a/src/core/file_sys/nup_container.cpp b/src/core/file_sys/nup_container.cpp new file mode 100644 index 0000000000..0e485b44ec --- /dev/null +++ b/src/core/file_sys/nup_container.cpp @@ -0,0 +1,676 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "core/file_sys/nup_container.h" + +#include <algorithm> +#include <cstddef> +#include <cstring> +#include <list> +#include <map> +#include <unordered_map> + +#include <openssl/evp.h> +#include <zstd.h> + +#include "common/logging.h" +#include "common/swap.h" +#include "core/file_sys/vfs/vfs_offset.h" + +namespace FileSys { +namespace { + +constexpr u32 NoDir = 0xFFFFFFFF; +constexpr std::size_t SuperblockSize = 0x80; +constexpr std::size_t PackageRecordSize = 0x40; +constexpr std::size_t DirRecordSize = 0x18; +constexpr std::size_t FileRecordSize = 0x50; +constexpr std::size_t TocHeaderSize = 0x30; +constexpr std::size_t TlvHeaderSize = 0x8; +constexpr u32 TocMagic = 0x01434F54; // "TOC\x01" little-endian +constexpr u8 CodecRaw = 0; +constexpr u8 CodecZstd = 1; +constexpr u8 PackageFlagStandaloneUpdate = 0x01; + +#pragma pack(push, 1) +struct Superblock { + std::array<u8, 4> magic; + u16_le header_size; + u16_le header_version; + u64_le toc_offset; + u64_le toc_size; + std::array<u8, 0x20> toc_sha256; + u64_le blob_offset; + u64_le file_size; + u32_le package_count; + u32_le flags; + std::array<u8, 0x10> uuid; + std::array<u8, 0x20> reserved; +}; +static_assert(sizeof(Superblock) == SuperblockSize); + +struct TocHeader { + u32_le magic; + u32_le package_count; + u64_le dir_count; + u64_le file_count; + u64_le pool_offset; + u64_le pool_size; + u64_le reserved; +}; +static_assert(sizeof(TocHeader) == TocHeaderSize); + +struct PackageRecord { + u64_le title_id; // 0x00 + u32_le root_dir; // 0x08 + u32_le exefs_dir; // 0x0C + u32_le romfs_dir; // 0x10 + u32_le version; // 0x14 + u8 kind; // 0x18 + u8 flags; // 0x19 + u16_le reserved; // 0x1A + u32_le reserved2; // 0x1C + u64_le meta_offset; // 0x20 + u32_le meta_size; // 0x28 + std::array<u8, 20> reserved3; // 0x2C +}; +static_assert(sizeof(PackageRecord) == PackageRecordSize); +static_assert(offsetof(PackageRecord, title_id) == 0x00); +static_assert(offsetof(PackageRecord, root_dir) == 0x08); +static_assert(offsetof(PackageRecord, exefs_dir) == 0x0C); +static_assert(offsetof(PackageRecord, romfs_dir) == 0x10); +static_assert(offsetof(PackageRecord, version) == 0x14); +static_assert(offsetof(PackageRecord, kind) == 0x18); +static_assert(offsetof(PackageRecord, flags) == 0x19); +static_assert(offsetof(PackageRecord, reserved) == 0x1A); +static_assert(offsetof(PackageRecord, reserved2) == 0x1C); +static_assert(offsetof(PackageRecord, meta_offset) == 0x20); +static_assert(offsetof(PackageRecord, meta_size) == 0x28); + +struct DirRecord { + u32_le name_offset; + u32_le name_length; + u32_le first_child; + u32_le child_count; + u64_le reserved; +}; +static_assert(sizeof(DirRecord) == DirRecordSize); + +struct FileRecord { + u32_le name_offset; // 0x00 + u32_le name_length; // 0x04 + u64_le size; // 0x08 + u64_le extent_offset; // 0x10 + u64_le extent_size; // 0x18 + std::array<u8, 0x20> sha256; // 0x20 + u8 codec; // 0x40 + u8 codec_flags; // 0x41 + u16_le reserved2; // 0x42 + std::array<u8, 12> reserved3; // 0x44 +}; +static_assert(sizeof(FileRecord) == FileRecordSize); +static_assert(offsetof(FileRecord, name_offset) == 0x00); +static_assert(offsetof(FileRecord, name_length) == 0x04); +static_assert(offsetof(FileRecord, size) == 0x08); +static_assert(offsetof(FileRecord, extent_offset) == 0x10); +static_assert(offsetof(FileRecord, extent_size) == 0x18); +static_assert(offsetof(FileRecord, sha256) == 0x20); +static_assert(offsetof(FileRecord, codec) == 0x40); +static_assert(offsetof(FileRecord, codec_flags) == 0x41); +static_assert(offsetof(FileRecord, reserved2) == 0x42); + +struct TlvHeader { + u16_le type; + u16_le reserved; + u32_le size; +}; +static_assert(sizeof(TlvHeader) == TlvHeaderSize); +#pragma pack(pop) + +enum MetaType : u16 { + MetaNpdm = 0x0001, + MetaIcon = 0x0003, + MetaDisplayVersion = 0x0004, + MetaTitleName = 0x0005, + MetaLocalizedTitles = 0x0009, + MetaLocalizedIcons = 0x000A, +}; + +bool Sha256(const u8* data, std::size_t size, std::array<u8, 0x20>& out) { + unsigned int length = 0; + EVP_Digest(data, size, out.data(), &length, EVP_sha256(), nullptr); + return length == out.size(); +} + +/// A member file whose extent is zstd-compressed. Decompresses on first read +/// and caches the result (LRU-evicted) unless the file exceeds the cache cap. +class NupVfsFile final : public VfsFile { +public: + NupVfsFile(VirtualFile backing_, u64 extent_offset_, u64 extent_size_, u64 uncompressed_size_, + std::string name_, std::shared_ptr<NupDecompressCache> cache_, + std::size_t cache_index_) + : backing(std::move(backing_)), extent_offset(extent_offset_), extent_size(extent_size_), + uncompressed_size(uncompressed_size_), name(std::move(name_)), cache(std::move(cache_)), + cache_index(cache_index_), cache_enabled(uncompressed_size <= NupDecompressCache::DecompressCacheCap) {} + + std::string GetName() const override { + return name; + } + + std::size_t GetSize() const override { + return static_cast<std::size_t>(uncompressed_size); + } + + bool Resize(std::size_t) override { + return false; + } + + VirtualDir GetContainingDirectory() const override { + return nullptr; + } + + bool IsWritable() const override { + return false; + } + + bool IsReadable() const override { + return true; + } + + std::size_t Read(u8* data, std::size_t length, std::size_t offset) const override { + if (offset >= uncompressed_size) { + return 0; + } + length = std::min(length, static_cast<std::size_t>(uncompressed_size) - offset); + + { + std::scoped_lock lock{cache->mutex}; + if (const auto* cached = cache->Find(cache_index)) { + std::memcpy(data, cached->data() + offset, length); + return length; + } + } + + // Miss: decompress without holding the lock, then publish and copy. + std::vector<u8> scratch = Decompress(); + if (scratch.empty()) { + return 0; + } + { + std::scoped_lock lock{cache->mutex}; + if (cache_enabled) { + cache->Insert(cache_index, scratch); + } + if (const auto* cached = cache->Find(cache_index)) { + std::memcpy(data, cached->data() + offset, length); + return length; + } + } + std::memcpy(data, scratch.data() + offset, length); + return length; + } + + std::size_t Write(const u8*, std::size_t, std::size_t) override { + return 0; + } + + bool Rename(std::string_view new_name) override { + name = std::string(new_name); + return true; + } + +private: + std::vector<u8> Decompress() const { + if (extent_size > backing->GetSize() || extent_offset > backing->GetSize() - extent_size) { + return {}; + } + const auto compressed = backing->ReadBytes(static_cast<std::size_t>(extent_size), + static_cast<std::size_t>(extent_offset)); + if (compressed.size() != extent_size) { + return {}; + } + std::vector<u8> out(static_cast<std::size_t>(uncompressed_size)); + const std::size_t result = + ZSTD_decompress(out.data(), out.size(), compressed.data(), compressed.size()); + if (ZSTD_isError(result) || result != out.size()) { + LOG_ERROR(Loader, "NUP zstd decompression failed for {}", name); + return {}; + } + return out; + } + + VirtualFile backing; + u64 extent_offset; + u64 extent_size; + u64 uncompressed_size; + std::string name; + std::shared_ptr<NupDecompressCache> cache; + std::size_t cache_index; + bool cache_enabled; +}; + +} // namespace + + +bool NupContainer::IsNupFile(const VirtualFile& file) { + if (file == nullptr || file->GetSize() < SuperblockSize) { + return false; + } + std::array<u8, 4> magic{}; + file->ReadObject(&magic, 0); + return magic == HeaderMagic; +} + +NupContainer::NupContainer(VirtualFile file_) + : file(std::move(file_)), cache(std::make_shared<NupDecompressCache>()) { + if (file == nullptr) { + return; + } + if (Parse()) { + status = Loader::ResultStatus::Success; + } else if (status == Loader::ResultStatus::ErrorNullFile) { + status = Loader::ResultStatus::ErrorNUPBadFormat; + } +} + +NupContainer::~NupContainer() = default; + +bool NupContainer::Parse() { + if (file->GetSize() < SuperblockSize) { + status = Loader::ResultStatus::ErrorNUPBadFormat; + return false; + } + Superblock super{}; + if (file->ReadObject(&super, 0) != sizeof(Superblock)) { + status = Loader::ResultStatus::ErrorNUPBadFormat; + return false; + } + if (super.magic != HeaderMagic || super.header_size < SuperblockSize || super.flags != 0) { + status = Loader::ResultStatus::ErrorNUPBadFormat; + return false; + } + toc_offset = super.toc_offset; + toc_size = super.toc_size; + blob_offset = super.blob_offset; + file_size = super.file_size; + if (file_size != file->GetSize() || toc_offset < SuperblockSize || + blob_offset < toc_offset + toc_size || blob_offset % 0x10 != 0 || + blob_offset > file_size) { + status = Loader::ResultStatus::ErrorNUPBadFormat; + return false; + } + + const auto toc = file->ReadBytes(static_cast<std::size_t>(toc_size), + static_cast<std::size_t>(toc_offset)); + if (toc.size() != toc_size) { + status = Loader::ResultStatus::ErrorNUPBadFormat; + return false; + } + std::array<u8, 0x20> toc_hash{}; + if (!Sha256(toc.data(), toc.size(), toc_hash) || toc_hash != super.toc_sha256) { + LOG_ERROR(Loader, "NUP TOC SHA-256 mismatch, the file is corrupt or was modified"); + status = Loader::ResultStatus::ErrorNUPBadFormat; + return false; + } + + TocHeader header{}; + if (toc.size() < sizeof(header)) { + status = Loader::ResultStatus::ErrorNUPBadFormat; + return false; + } + std::memcpy(&header, toc.data(), sizeof(header)); + if (header.magic != TocMagic || header.package_count == 0 || + static_cast<u32>(header.package_count) != super.package_count) { + status = Loader::ResultStatus::ErrorNUPBadFormat; + return false; + } + + const u64 needed = TocHeaderSize + static_cast<u64>(header.package_count) * PackageRecordSize + + header.dir_count * DirRecordSize + header.file_count * FileRecordSize; + const u64 children_offset = needed; + const u64 children_bytes = header.pool_offset - needed; + if (toc.size() < needed || header.pool_offset < needed || + header.pool_offset + header.pool_size > toc.size() || children_bytes % sizeof(u32) != 0) { + status = Loader::ResultStatus::ErrorNUPBadFormat; + return false; + } + const u8* base = toc.data(); + + std::size_t pos = TocHeaderSize; + packages.reserve(header.package_count); + for (u32 i = 0; i < header.package_count; ++i) { + PackageRecord record{}; + std::memcpy(&record, base + pos, sizeof(record)); + pos += PackageRecordSize; + if (record.kind > static_cast<u8>(PackageKind::Addon) || + (record.flags & ~PackageFlagStandaloneUpdate) != 0 || + record.root_dir >= header.dir_count || + (record.exefs_dir != NoDir && record.exefs_dir >= header.dir_count) || + (record.romfs_dir != NoDir && record.romfs_dir >= header.dir_count)) { + status = Loader::ResultStatus::ErrorNUPBadFormat; + return false; + } + PackageInfo package; + package.title_id = record.title_id; + package.version = record.version; + package.kind = static_cast<PackageKind>(record.kind); + package.standalone_update = (record.flags & PackageFlagStandaloneUpdate) != 0; + package.root_dir = record.root_dir; + package.exefs_dir = record.exefs_dir; + package.romfs_dir = record.romfs_dir; + if (record.meta_size > 0 && record.meta_offset != 0 && + record.meta_offset + record.meta_size <= file_size) { + package.meta_offset = record.meta_offset; + package.meta_size = record.meta_size; + } + packages.push_back(std::move(package)); + } + + dirs.resize(header.dir_count); + for (u64 i = 0; i < header.dir_count; ++i) { + DirRecord record{}; + std::memcpy(&record, base + pos, sizeof(record)); + pos += DirRecordSize; + if (record.first_child + record.child_count > children_bytes / sizeof(u32) || + record.name_offset + record.name_length > header.pool_size) { + status = Loader::ResultStatus::ErrorNUPBadFormat; + return false; + } + dirs[i].name = std::string(reinterpret_cast<const char*>(base) + header.pool_offset + + record.name_offset, + record.name_length); + dirs[i].first_child = record.first_child; + dirs[i].child_count = record.child_count; + } + + files.resize(header.file_count); + for (u64 i = 0; i < header.file_count; ++i) { + FileRecord record{}; + std::memcpy(&record, base + pos, sizeof(record)); + pos += FileRecordSize; + if (record.codec > CodecZstd || record.codec_flags != 0 || + (record.size == 0 && (record.extent_offset != 0 || record.extent_size != 0)) || + (record.size != 0 && + (record.extent_size == 0 || record.extent_offset < blob_offset || + record.extent_offset % 0x10 != 0 || + record.extent_offset + record.extent_size > file_size)) || + record.name_offset + record.name_length > header.pool_size) { + status = Loader::ResultStatus::ErrorNUPBadFormat; + return false; + } + files[i].name = std::string(reinterpret_cast<const char*>(base) + header.pool_offset + + record.name_offset, + record.name_length); + files[i].size = record.size; + files[i].sha256 = record.sha256; + files[i].extent_offset = record.extent_offset; + files[i].extent_size = record.extent_size; + files[i].codec = record.codec; + } + + children.resize(children_bytes / sizeof(u32)); + std::memcpy(children.data(), base + children_offset, children_bytes); + for (const u32 child : children) { + if (child >= header.dir_count + header.file_count) { + status = Loader::ResultStatus::ErrorNUPBadFormat; + return false; + } + } + + for (auto& package : packages) { + if (!ParseMeta(package)) { + status = Loader::ResultStatus::ErrorNUPBadFormat; + return false; + } + } + return true; +} + +bool NupContainer::ParseMeta(PackageInfo& package) { + if (package.meta_offset == 0) { + return true; + } + const auto block = file->ReadBytes(static_cast<std::size_t>(package.meta_size), + static_cast<std::size_t>(package.meta_offset)); + if (block.size() != package.meta_size) { + return false; + } + std::size_t pos = 0; + while (pos + TlvHeaderSize <= block.size()) { + TlvHeader header{}; + std::memcpy(&header, block.data() + pos, sizeof(header)); + pos += TlvHeaderSize; + if (pos + header.size > block.size()) { + return false; + } + const u8* payload = block.data() + pos; + pos += header.size; + switch (header.type) { + case MetaNpdm: + package.npdm.assign(payload, payload + header.size); + break; + case MetaIcon: + package.icon.assign(payload, payload + header.size); + break; + case MetaTitleName: + package.title_name = std::string(reinterpret_cast<const char*>(payload), header.size); + break; + case MetaDisplayVersion: + package.display_version = + std::string(reinterpret_cast<const char*>(payload), header.size); + break; + case MetaLocalizedTitles: + case MetaLocalizedIcons: { + const bool is_title = header.type == MetaLocalizedTitles; + std::size_t record_pos = 0; + while (record_pos < header.size) { + const u8 lang_length = payload[record_pos++]; + if (record_pos + lang_length + sizeof(u32) > header.size) { + return false; + } + const std::string lang(reinterpret_cast<const char*>(payload) + record_pos, + lang_length); + record_pos += lang_length; + u32 data_length = 0; + std::memcpy(&data_length, payload + record_pos, sizeof(data_length)); + record_pos += sizeof(data_length); + if (record_pos + data_length > header.size) { + return false; + } + if (is_title) { + package.localized_titles.emplace_back( + lang, std::string(reinterpret_cast<const char*>(payload) + record_pos, + data_length)); + } else { + package.localized_icons.emplace_back( + lang, std::vector<u8>(payload + record_pos, + payload + record_pos + data_length)); + } + record_pos += data_length; + } + break; + } + default: + break; + } + } + return true; +} + +std::optional<std::size_t> NupContainer::FindPackage(PackageKind kind, + std::optional<u64> title_id) const { + for (std::size_t i = 0; i < packages.size(); ++i) { + if (packages[i].kind != kind) { + continue; + } + if (title_id && packages[i].title_id != *title_id) { + continue; + } + return i; + } + return std::nullopt; +} + +std::optional<std::size_t> NupContainer::FindBasePackage() const { + return FindPackage(PackageKind::Base); +} + +std::optional<std::size_t> NupContainer::FindBestUpdatePackage() const { + std::optional<std::size_t> best; + for (std::size_t i = 0; i < packages.size(); ++i) { + if (packages[i].kind != PackageKind::Update) { + continue; + } + if (!best || packages[i].version > packages[*best].version) { + best = i; + } + } + return best; +} + +VirtualFile NupContainer::OpenFile(std::size_t file_index) { + if (file_index >= files.size()) { + return nullptr; + } + const FileNode& node = files[file_index]; + if (node.size == 0) { + return std::make_shared<VectorVfsFile>(std::vector<u8>{}, node.name); + } + if (node.codec == CodecRaw) { + return std::make_shared<OffsetVfsFile>( + file, static_cast<std::size_t>(node.extent_size), + static_cast<std::size_t>(node.extent_offset), node.name); + } + return std::make_shared<NupVfsFile>(file, node.extent_offset, node.extent_size, node.size, + node.name, cache, file_index); +} + +void NupContainer::BuildDirTree(u32 dir_index, VectorVfsDirectory& out) { + const DirNode& node = dirs[dir_index]; + for (u32 slot = node.first_child; slot < node.first_child + node.child_count; ++slot) { + const u32 child = children[slot]; + if (child >= dirs.size()) { + out.AddFile(OpenFile(child - static_cast<u32>(dirs.size()))); + } else { + auto sub = std::make_shared<VectorVfsDirectory>(std::vector<VirtualFile>{}, + std::vector<VirtualDir>{}, + dirs[child].name); + BuildDirTree(child, *sub); + out.AddDirectory(sub); + } + } +} + +VirtualDir NupContainer::BuildPackageDir(u32 dir_index) { + auto out = std::make_shared<VectorVfsDirectory>(std::vector<VirtualFile>{}, + std::vector<VirtualDir>{}, dirs[dir_index].name); + BuildDirTree(dir_index, *out); + return out; +} + +VirtualDir NupContainer::GetExeFSDir(std::size_t package_index) { + if (package_index >= packages.size() || packages[package_index].exefs_dir == NoDir) { + return nullptr; + } + if (exefs_dir_cache.size() <= package_index) { + exefs_dir_cache.resize(packages.size()); + } + if (exefs_dir_cache[package_index] == nullptr) { + exefs_dir_cache[package_index] = BuildPackageDir(packages[package_index].exefs_dir); + } + return exefs_dir_cache[package_index]; +} + +VirtualDir NupContainer::GetRomFSDir(std::size_t package_index) { + if (package_index >= packages.size() || packages[package_index].romfs_dir == NoDir) { + return nullptr; + } + if (romfs_dir_cache.size() <= package_index) { + romfs_dir_cache.resize(packages.size()); + } + if (romfs_dir_cache[package_index] == nullptr) { + romfs_dir_cache[package_index] = BuildPackageDir(packages[package_index].romfs_dir); + } + return romfs_dir_cache[package_index]; +} + +VirtualDir NupContainer::MergeShadowDirs(const VirtualDir& base, const VirtualDir& overlay) { + if (base == nullptr) { + return overlay; + } + if (overlay == nullptr) { + return base; + } + + std::map<std::string, VirtualFile, std::less<>> file_map; + for (const auto& entry : base->GetFiles()) { + file_map.insert_or_assign(entry->GetName(), entry); + } + for (const auto& entry : overlay->GetFiles()) { + file_map.insert_or_assign(entry->GetName(), entry); + } + + std::map<std::string, VirtualDir, std::less<>> dir_map; + for (const auto& entry : base->GetSubdirectories()) { + dir_map.insert_or_assign(entry->GetName(), entry); + } + for (const auto& entry : overlay->GetSubdirectories()) { + const auto it = dir_map.find(entry->GetName()); + if (it != dir_map.end()) { + dir_map.insert_or_assign(entry->GetName(), MergeShadowDirs(it->second, entry)); + } else { + dir_map.insert_or_assign(entry->GetName(), entry); + } + } + + std::vector<VirtualFile> merged_files; + merged_files.reserve(file_map.size()); + for (const auto& [_, entry] : file_map) { + merged_files.push_back(entry); + } + std::vector<VirtualDir> merged_dirs; + merged_dirs.reserve(dir_map.size()); + for (const auto& [_, entry] : dir_map) { + merged_dirs.push_back(entry); + } + return std::make_shared<VectorVfsDirectory>(std::move(merged_files), std::move(merged_dirs), + base->GetName()); +} + +std::vector<u8> NupContainer::ReadExtentRaw(const FileNode& node) const { + return file->ReadBytes(static_cast<std::size_t>(node.extent_size), + static_cast<std::size_t>(node.extent_offset)); +} + +Loader::ResultStatus NupContainer::VerifyIntegrity( + std::function<bool(std::size_t, std::size_t)> progress_callback) const { + const std::size_t total = files.size(); + for (std::size_t i = 0; i < total; ++i) { + const FileNode& node = files[i]; + std::vector<u8> content; + if (node.size == 0) { + content = ReadExtentRaw(node); + } else if (node.codec == CodecRaw) { + content = ReadExtentRaw(node); + } else { + NupVfsFile view(file, node.extent_offset, node.extent_size, node.size, node.name, + cache, i); + content = view.ReadAllBytes(); + } + if (content.size() != node.size) { + LOG_ERROR(Loader, "NUP integrity check failed for {}: short read", node.name); + return Loader::ResultStatus::ErrorIntegrityVerificationFailed; + } + std::array<u8, 0x20> hash{}; + if (!Sha256(content.data(), content.size(), hash) || hash != node.sha256) { + LOG_ERROR(Loader, "NUP integrity check failed for {}: SHA-256 mismatch", node.name); + return Loader::ResultStatus::ErrorIntegrityVerificationFailed; + } + if (progress_callback && !progress_callback(i + 1, total)) { + return Loader::ResultStatus::ErrorIntegrityVerificationFailed; + } + } + return Loader::ResultStatus::Success; +} + +} // namespace FileSys diff --git a/src/core/file_sys/nup_container.h b/src/core/file_sys/nup_container.h new file mode 100644 index 0000000000..118b137a7b --- /dev/null +++ b/src/core/file_sys/nup_container.h @@ -0,0 +1,177 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <array> +#include <functional> +#include <list> +#include <memory> +#include <mutex> +#include <optional> +#include <string> +#include <unordered_map> +#include <vector> + +#include "common/common_types.h" +#include "core/file_sys/vfs/vfs_types.h" +#include "core/file_sys/vfs/vfs_vector.h" +#include "core/loader/loader.h" + +namespace FileSys { + +/// LRU over decompressed zstd extents, shared by the files of one container. +struct NupDecompressCache { + std::mutex mutex; + std::list<std::pair<std::size_t, std::vector<u8>>> lru; + std::unordered_map<std::size_t, std::list<std::pair<std::size_t, std::vector<u8>>>::iterator> + index; + std::size_t bytes{}; + + std::vector<u8>* Find(std::size_t file_index) { + const auto it = index.find(file_index); + if (it == index.end()) { + return nullptr; + } + lru.splice(lru.begin(), lru, it->second); + return &it->second->second; + } + + void Insert(std::size_t file_index, std::vector<u8>& data) { + while (bytes + data.size() > DecompressCacheCap && !lru.empty()) { + bytes -= lru.back().second.size(); + index.erase(lru.back().first); + lru.pop_back(); + } + if (bytes + data.size() > DecompressCacheCap) { + return; + } + lru.emplace_front(file_index, data); + index.emplace(file_index, lru.begin()); + bytes += data.size(); + } + + static constexpr std::size_t DecompressCacheCap = 512 * 1024 * 1024; +}; + +/** + * NUP (NX Package) container: stores loose ExeFS/RomFS trees for a base program, + * its updates and add-on content in one file. Layout: + * + * Superblock (0x80 bytes, magic "NUP\x01") + * Table of Contents (packages, dir/file nodes, children array, string pool) + * Blob region (per-file extents, raw or zstd codec, 16-byte aligned) + * + * Per-package meta TLVs carry NPDM bytes, title names (default + localized), + * icons and a display version. All integers little-endian. + */ +class NupContainer { +public: + static constexpr std::array<u8, 4> HeaderMagic{'N', 'U', 'P', 0x01}; + + enum class PackageKind : u8 { + Base = 0, + Update = 1, + Addon = 2, + }; + + + struct PackageInfo { + u64 title_id{}; + u32 version{}; + PackageKind kind{}; + bool standalone_update{}; + u32 root_dir{}; + u32 exefs_dir{}; + u32 romfs_dir{}; + u64 meta_offset{}; + u32 meta_size{}; + + std::vector<u8> npdm; + std::string title_name; + std::vector<std::pair<std::string, std::string>> localized_titles; + std::vector<u8> icon; + std::vector<std::pair<std::string, std::vector<u8>>> localized_icons; + std::string display_version; + }; + + /// Quick magic check without a full parse. + static bool IsNupFile(const VirtualFile& file); + + /// Recursively merges overlay into a copy of base; overlay paths win. + static VirtualDir MergeShadowDirs(const VirtualDir& base, const VirtualDir& overlay); + + explicit NupContainer(VirtualFile file_); + ~NupContainer(); + + Loader::ResultStatus GetStatus() const { + return status; + } + + const std::vector<PackageInfo>& GetPackages() const { + return packages; + } + + std::optional<std::size_t> FindPackage(PackageKind kind, + std::optional<u64> title_id = std::nullopt) const; + std::optional<std::size_t> FindBasePackage() const; + /// Highest version among Update packages, if any. + std::optional<std::size_t> FindBestUpdatePackage() const; + + /// Opens a member file by node index; raw extents are zero-copy views into + /// the container, zstd extents are lazily decompressed and cached. + VirtualFile OpenFile(std::size_t file_index); + + VirtualDir GetExeFSDir(std::size_t package_index); + VirtualDir GetRomFSDir(std::size_t package_index); + + /// Stream-hashes every member file against its stored SHA-256. + Loader::ResultStatus VerifyIntegrity( + std::function<bool(std::size_t, std::size_t)> progress_callback) const; + + std::size_t GetFileCount() const { + return files.size(); + } + +private: + struct DirNode { + std::string name; + u32 first_child{}; + u32 child_count{}; + }; + + struct FileNode { + std::string name; + u64 size{}; + std::array<u8, 0x20> sha256{}; + u64 extent_offset{}; + u64 extent_size{}; + u8 codec{}; + }; + + bool Parse(); + bool ParseMeta(PackageInfo& package); + void BuildDirTree(u32 dir_index, VectorVfsDirectory& out); + VirtualDir BuildPackageDir(u32 dir_index); + std::vector<u8> ReadExtentRaw(const FileNode& node) const; + + VirtualFile file; + Loader::ResultStatus status{Loader::ResultStatus::ErrorNullFile}; + + u64 toc_offset{}; + u64 toc_size{}; + u64 blob_offset{}; + u64 file_size{}; + + std::vector<DirNode> dirs; + std::vector<FileNode> files; + std::vector<u32> children; + std::vector<PackageInfo> packages; + + std::vector<VirtualDir> exefs_dir_cache; + std::vector<VirtualDir> romfs_dir_cache; + + std::shared_ptr<NupDecompressCache> cache; +}; + +} // namespace FileSys diff --git a/src/core/file_sys/nup_writer.cpp b/src/core/file_sys/nup_writer.cpp new file mode 100644 index 0000000000..8c7728ed01 --- /dev/null +++ b/src/core/file_sys/nup_writer.cpp @@ -0,0 +1,1459 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "core/file_sys/nup_writer.h" + +#include <algorithm> +#include <array> +#include <cctype> +#include <charconv> +#include <cstring> +#include <limits> +#include <map> +#include <memory> +#include <optional> +#include <span> +#include <stdexcept> +#include <unordered_map> +#include <unordered_set> +#include <vector> + +#include <nlohmann/json.hpp> +#include <openssl/evp.h> +#include <zstd.h> + +#include "common/fs/file.h" +#include "common/fs/path_util.h" +#include "core/file_sys/control_metadata.h" +#include "core/file_sys/vfs/vfs_real.h" +#include "core/loader/deconstructed_rom_directory.h" + +namespace FileSys::Nup { +namespace { + +using Json = nlohmann::json; +namespace fs = std::filesystem; + +constexpr std::array<u8, 4> Magic{'N', 'U', 'P', 0x01}; +constexpr std::array<u8, 4> TocMagic{'T', 'O', 'C', 0x01}; +constexpr u64 SuperblockSize = 0x80; +constexpr u64 TocHeaderSize = 0x30; +constexpr u64 PackageRecordSize = 0x40; +constexpr u64 DirRecordSize = 0x18; +constexpr u64 FileRecordSize = 0x50; +constexpr u64 ExtentAlignment = 16; +constexpr u32 NoDir = 0xFFFFFFFF; + +constexpr u8 KindBase = 0; +constexpr u8 KindUpdate = 1; +constexpr u8 KindAddon = 2; +constexpr u8 FlagStandaloneUpdate = 0x01; +constexpr u8 CodecRaw = 0; +constexpr u8 CodecZstd = 1; + +constexpr u16 TlvNpdm = 0x0001; +constexpr u16 TlvNacp = 0x0002; +constexpr u16 TlvIcon = 0x0003; +constexpr u16 TlvDisplayVersion = 0x0004; +constexpr u16 TlvTitleName = 0x0005; +constexpr u16 TlvPublisher = 0x0006; +constexpr u16 TlvSourceDump = 0x0008; + +constexpr int JournalVersion = 1; +constexpr u64 ProgressStep = 64ULL * 1024 * 1024; +constexpr size_t MaxChunkFiles = 4096; +constexpr size_t ReadBufferSize = 4 * 1024 * 1024; + +class PackError : public std::runtime_error { +public: + PackError(PackResult result_, const std::string& message) + : std::runtime_error(message), result{result_} {} + PackResult result; +}; + +u64 AlignUp(u64 value, u64 alignment) { + return (value + alignment - 1) / alignment * alignment; +} + +std::string ToUtf8(const fs::path& path) { + const auto u8 = path.generic_u8string(); + return {u8.begin(), u8.end()}; +} + +fs::path FromUtf8(const std::string& value) { + return fs::path{std::u8string{value.begin(), value.end()}}; +} + +std::string Lower(std::string value) { + std::ranges::transform(value, value.begin(), + [](unsigned char c) { return static_cast<char>(std::tolower(c)); }); + return value; +} + +fs::path WithSuffix(fs::path path, std::string_view suffix) { + path += suffix; + return path; +} + +bool IsFile(const fs::path& path) { + std::error_code ec; + return fs::is_regular_file(path, ec); +} + +fs::path FindDirectory(const fs::path& parent, std::string_view name) { + std::error_code ec; + if (parent.empty() || !fs::is_directory(parent, ec)) { + return {}; + } + const auto wanted = Lower(std::string{name}); + for (const auto& entry : fs::directory_iterator(parent, ec)) { + if (entry.is_directory(ec) && Lower(ToUtf8(entry.path().filename())) == wanted) { + return entry.path(); + } + } + return {}; +} + +std::optional<u64> ParseDecimal(const std::string& name) { + u64 value{}; + const auto* end = name.data() + name.size(); + const auto [ptr, ec] = std::from_chars(name.data(), end, value); + if (name.empty() || ec != std::errc{} || ptr != end) { + return std::nullopt; + } + return value; +} + +// The exefs folder a dump boots from: the newest updates/<version>, else base, else the root. +// Each must hold main, main.npdm and a romfs folder next to it or under base. +bool HasBootableExeFS(const fs::path& root) { + const auto bootable = [](const fs::path& exefs) { + return !exefs.empty() && IsFile(exefs / "main") && IsFile(exefs / "main.npdm"); + }; + fs::path exefs; + std::optional<u64> best_version; + if (const auto updates = FindDirectory(root, "updates"); !updates.empty()) { + std::error_code ec; + for (const auto& entry : fs::directory_iterator(updates, ec)) { + const auto version = ParseDecimal(ToUtf8(entry.path().filename())); + const auto candidate = FindDirectory(entry.path(), "exefs"); + if (version && bootable(candidate) && (!best_version || *version > *best_version)) { + best_version = version; + exefs = candidate; + } + } + } + const auto base = FindDirectory(root, "base"); + if (exefs.empty()) { + exefs = FindDirectory(base, "exefs"); + } + if (exefs.empty()) { + exefs = FindDirectory(root, "exefs"); + } + if (!bootable(exefs)) { + return false; + } + return !FindDirectory(exefs.parent_path(), "romfs").empty() || + !FindDirectory(base, "romfs").empty(); +} + +std::optional<u64> ParseTitleId(const std::string& name) { + u64 value{}; + const auto* end = name.data() + name.size(); + const auto [ptr, ec] = std::from_chars(name.data(), end, value, 16); + if (name.size() != 16 || ec != std::errc{} || ptr != end) { + return std::nullopt; + } + return value; +} + +std::string ToHex(std::span<const u8> data) { + static constexpr char digits[] = "0123456789abcdef"; + std::string out; + out.reserve(data.size() * 2); + for (const u8 byte : data) { + out += digits[byte >> 4]; + out += digits[byte & 0xF]; + } + return out; +} + +std::vector<u8> FromHex(const std::string& hex) { + if (hex.size() % 2 != 0) { + throw PackError(PackResult::IoError, "malformed journal"); + } + std::vector<u8> out(hex.size() / 2); + for (size_t i = 0; i < out.size(); ++i) { + const auto [ptr, ec] = std::from_chars(hex.data() + i * 2, hex.data() + i * 2 + 2, out[i], 16); + if (ec != std::errc{} || ptr != hex.data() + i * 2 + 2) { + throw PackError(PackResult::IoError, "malformed journal"); + } + } + return out; +} + +std::vector<u8> ReadWholeFile(const fs::path& path) { + Common::FS::IOFile in{path, Common::FS::FileAccessMode::Read}; + if (!in.IsOpen()) { + return {}; + } + std::vector<u8> data(in.GetSize()); + if (in.ReadSpan(std::span{data}) != data.size()) { + return {}; + } + return data; +} + +// Same as pack.py: the program ID at ACI0 + 0x10, whatever the NPDM header revision. +std::optional<u64> NpdmProgramId(std::span<const u8> npdm) { + static constexpr std::array<u8, 4> aci0{'A', 'C', 'I', '0'}; + const auto it = std::search(npdm.begin(), npdm.end(), aci0.begin(), aci0.end()); + const auto offset = static_cast<size_t>(it - npdm.begin()); + if (it == npdm.end() || offset + 0x18 > npdm.size()) { + return std::nullopt; + } + u64 id = 0; + for (size_t i = 0; i < 8; ++i) { + id |= static_cast<u64>(npdm[offset + 0x10 + i]) << (8 * i); + } + return id; +} + +void PutLE(std::vector<u8>& buffer, size_t offset, u64 value, size_t bytes) { + for (size_t i = 0; i < bytes; ++i) { + buffer[offset + i] = static_cast<u8>(value >> (8 * i)); + } +} + +void PutBytes(std::vector<u8>& buffer, size_t offset, std::span<const u8> bytes) { + std::memcpy(buffer.data() + offset, bytes.data(), bytes.size()); +} + +using Digest = std::array<u8, 32>; + +class Sha256 { +public: + Sha256() : ctx{EVP_MD_CTX_new()} { + if (!ctx) { + throw PackError(PackResult::IoError, "cannot initialise SHA-256"); + } + Reset(); + } + ~Sha256() { + EVP_MD_CTX_free(ctx); + } + Sha256(const Sha256&) = delete; + Sha256& operator=(const Sha256&) = delete; + + void Reset() { + if (EVP_DigestInit_ex(ctx, EVP_sha256(), nullptr) != 1) { + throw PackError(PackResult::IoError, "cannot initialise SHA-256"); + } + } + + void Update(const void* data, size_t size) { + EVP_DigestUpdate(ctx, data, size); + } + + Digest Final() { + Digest digest{}; + unsigned int length{}; + EVP_DigestFinal_ex(ctx, digest.data(), &length); + return digest; + } + +private: + EVP_MD_CTX* ctx; +}; + +class ZstdContext { +public: + // Archives must be byte-identical across platforms, so compression is always + // single-threaded (zstd's multithreaded frames differ) and every frame parameter is set + // explicitly instead of relying on library defaults. + explicit ZstdContext(int level) : cctx{ZSTD_createCCtx()} { + if (!cctx || ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level)) || + ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0)) || + ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 1)) || + ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0)) || + ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_dictIDFlag, 0))) { + throw PackError(PackResult::IoError, "cannot initialise zstd"); + } + } + ~ZstdContext() { + ZSTD_freeCCtx(cctx); + } + ZstdContext(const ZstdContext&) = delete; + ZstdContext& operator=(const ZstdContext&) = delete; + + // The pledged size makes zstd record the content size in the frame header, which the + // format requires. + void Begin(u64 size) { + if (ZSTD_isError(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only)) || + ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(cctx, size))) { + throw PackError(PackResult::IoError, "cannot start zstd frame"); + } + } + + ZSTD_CCtx* get() const { + return cctx; + } + +private: + ZSTD_CCtx* cctx; +}; + +// One package and the metadata it is written with. The plan is everything that must survive a +// low-storage run deleting its sources, so it is what the journal stores. +struct PlanPackage { + u8 kind{}; + u8 flags{}; + u64 title_id{}; + u32 version{}; + std::vector<u8> meta; +}; + +struct PlanEntry { + u32 package{}; + std::string path; // inside the package tree, e.g. "romfs/Data/file.bin" + std::string source; // relative to the dump root + bool directory{}; +}; + +struct Plan { + std::vector<PlanPackage> packages; + std::vector<PlanEntry> entries; +}; + +struct Extent { + u64 size{}; + Digest sha256{}; + u64 offset{}; + u64 stored_size{}; + u8 codec{}; +}; + +struct Layout { + struct Dir { + std::string name; + u32 first_child{}; + u32 child_count{}; + }; + struct File { + std::string name; + size_t entry{}; + }; + struct Package { + u32 root_dir{}; + u32 exefs_dir = NoDir; + u32 romfs_dir = NoDir; + }; + std::vector<Dir> dirs; + std::vector<File> files; + std::vector<u64> children; // FileTag | file index, or dir index + std::vector<Package> packages; + std::string pool; + std::unordered_map<std::string, u32> pool_index; + u64 toc_size{}; + u64 blob_offset{}; + + static constexpr u64 FileTag = 1ULL << 32; + + u32 Intern(const std::string& name) { + const auto [it, inserted] = pool_index.try_emplace(name, static_cast<u32>(pool.size())); + if (inserted) { + pool += name; + } + return it->second; + } +}; + +// Children sorted by the bytes of their names: std::string compares as unsigned char. +struct TreeDir { + struct Child { + std::unique_ptr<TreeDir> dir; + size_t file_entry{}; + }; + std::string name; + std::map<std::string, Child> children; + + TreeDir* Descend(const std::string& component) { + auto& child = children[component]; + if (!child.dir) { + child.dir = std::make_unique<TreeDir>(); + child.dir->name = component; + } + return child.dir.get(); + } +}; + +// Lays out nodes as pack.py does: directories in pre-order, each directory's child slots +// contiguous, its files numbered in slot order before its subdirectories are visited. +u32 AssignDir(TreeDir& dir, Layout& layout) { + const auto id = static_cast<u32>(layout.dirs.size()); + layout.dirs.push_back({dir.name, static_cast<u32>(layout.children.size()), + static_cast<u32>(dir.children.size())}); + const size_t slot_base = layout.children.size(); + layout.children.resize(slot_base + dir.children.size()); + size_t slot = slot_base; + for (auto& [name, child] : dir.children) { + if (!child.dir) { + layout.children[slot] = Layout::FileTag | layout.files.size(); + layout.files.push_back({name, child.file_entry}); + } + ++slot; + } + slot = slot_base; + for (auto& [name, child] : dir.children) { + if (child.dir) { + layout.children[slot] = AssignDir(*child.dir, layout); + } + ++slot; + } + return id; +} + +Layout BuildLayout(const Plan& plan) { + std::vector<TreeDir> roots(plan.packages.size()); + for (size_t i = 0; i < plan.entries.size(); ++i) { + const auto& entry = plan.entries[i]; + TreeDir* dir = &roots.at(entry.package); + size_t start = 0; + while (true) { + const auto slash = entry.path.find('/', start); + const auto component = entry.path.substr(start, slash - start); + if (slash == std::string::npos) { + if (entry.directory) { + dir->Descend(component); + } else { + auto& child = dir->children[component]; + if (child.dir) { + throw PackError(PackResult::InvalidSource, + "a file and a folder share the name " + entry.path); + } + child.file_entry = i; + } + break; + } + dir = dir->Descend(component); + start = slash + 1; + } + } + + Layout layout; + for (auto& root : roots) { + Layout::Package package; + package.root_dir = AssignDir(root, layout); + const auto& root_dir = layout.dirs[package.root_dir]; + for (u32 slot = root_dir.first_child; slot < root_dir.first_child + root_dir.child_count; + ++slot) { + const u64 child = layout.children[slot]; + if (child & Layout::FileTag) { + continue; + } + const auto id = static_cast<u32>(child); + if (layout.dirs[id].name == "exefs") { + package.exefs_dir = id; + } else if (layout.dirs[id].name == "romfs") { + package.romfs_dir = id; + } + } + layout.packages.push_back(package); + } + + if (layout.dirs.size() + layout.files.size() >= NoDir) { + throw PackError(PackResult::InvalidSource, "the dump has too many files"); + } + for (const auto& dir : layout.dirs) { + layout.Intern(dir.name); + } + for (const auto& file : layout.files) { + layout.Intern(file.name); + } + if (layout.pool.size() > std::numeric_limits<u32>::max()) { + throw PackError(PackResult::InvalidSource, "the dump's file names are too long"); + } + + const u64 unpadded = TocHeaderSize + plan.packages.size() * PackageRecordSize + + layout.dirs.size() * DirRecordSize + layout.files.size() * FileRecordSize + + layout.children.size() * sizeof(u32) + layout.pool.size(); + layout.toc_size = AlignUp(unpadded, ExtentAlignment); + layout.blob_offset = AlignUp(SuperblockSize + layout.toc_size, ExtentAlignment); + return layout; +} + +std::vector<u8> SerializeTlvs(std::vector<std::pair<u16, std::vector<u8>>> records) { + std::ranges::stable_sort(records, {}, &std::pair<u16, std::vector<u8>>::first); + std::vector<u8> out; + for (const auto& [type, payload] : records) { + const size_t at = out.size(); + out.resize(at + 8 + payload.size()); + PutLE(out, at, type, 2); + PutLE(out, at + 4, payload.size(), 4); + PutBytes(out, at + 8, payload); + } + return out; +} + +std::vector<u8> StringBytes(const std::string& value) { + return {value.begin(), value.end()}; +} + +// The dump's own loader supplies control data and the display version, the same way a loose +// dump reports them when it boots (control.nacp beside it, else the RomFS's version tag). +std::unique_ptr<Loader::AppLoader_DeconstructedRomDirectory> OpenLoader( + RealVfsFilesystem& vfs, const fs::path& package_dir) { + const auto exefs = FindDirectory(package_dir, "exefs"); + if (exefs.empty() || !IsFile(exefs / "main")) { + return nullptr; + } + auto main = vfs.OpenFile(Common::FS::PathToUTF8String(exefs / "main"), OpenMode::Read); + if (!main) { + return nullptr; + } + return std::make_unique<Loader::AppLoader_DeconstructedRomDirectory>(std::move(main)); +} + +class Discovery { +public: + Discovery(const fs::path& dump_root_, std::vector<fs::path> excluded_) + : dump_root{dump_root_}, excluded{std::move(excluded_)} {} + + Plan Run() { + const auto base = FindDirectory(dump_root, "base"); + const auto updates = FindDirectory(dump_root, "updates"); + const auto dlc = FindDirectory(dump_root, "dlc"); + + std::optional<u64> base_title_id; + if (!base.empty() && HasContent(base)) { + base_title_id = AddPackage(KindBase, base, 0, {}); + } else if (updates.empty() && HasContent(dump_root)) { + base_title_id = AddPackage(KindBase, dump_root, 0, {"base", "updates", "dlc"}); + } + + std::vector<std::pair<u64, fs::path>> versions; + std::error_code ec; + if (!updates.empty()) { + for (const auto& entry : fs::directory_iterator(updates, ec)) { + const auto version = ParseDecimal(ToUtf8(entry.path().filename())); + if (version && entry.is_directory(ec) && HasContent(entry.path())) { + if (*version > std::numeric_limits<u32>::max()) { + throw PackError(PackResult::InvalidSource, + "update version out of range: " + ToUtf8(entry.path())); + } + versions.emplace_back(*version, entry.path()); + } + } + } + std::ranges::sort(versions); + const u8 update_flags = base_title_id ? 0 : FlagStandaloneUpdate; + for (const auto& [version, path] : versions) { + AddPackage(KindUpdate, path, static_cast<u32>(version), {}, update_flags, + base_title_id.value_or(0)); + } + + if (!dlc.empty()) { + std::vector<std::pair<std::string, fs::path>> addons; + for (const auto& title : fs::directory_iterator(dlc, ec)) { + const auto title_name = ToUtf8(title.path().filename()); + if (!ParseTitleId(title_name) || !title.is_directory(ec)) { + continue; + } + for (const auto& content : fs::directory_iterator(title.path(), ec)) { + if (content.is_directory(ec) && HasContent(content.path())) { + addons.emplace_back(title_name + "/" + ToUtf8(content.path().filename()), + content.path()); + } + } + } + std::ranges::sort(addons); + for (const auto& [name, path] : addons) { + AddPackage(KindAddon, path, 0, {}, 0, *ParseTitleId(name.substr(0, 16))); + } + } + + if (plan.packages.empty()) { + throw PackError(PackResult::InvalidSource, "no packages found in the dump"); + } + AddApplicationMetadata(); + for (size_t i = 0; i < plan.packages.size(); ++i) { + auto& tlvs = package_tlvs[i]; + tlvs.emplace_back(TlvSourceDump, StringBytes(ToUtf8(dump_root.filename()))); + plan.packages[i].meta = SerializeTlvs(std::move(tlvs)); + } + return std::move(plan); + } + +private: + static bool HasContent(const fs::path& dir) { + return !FindDirectory(dir, "exefs").empty() || !FindDirectory(dir, "romfs").empty(); + } + + // Returns the package's title ID: the NPDM program ID, else the folder's or base's. + u64 AddPackage(u8 kind, const fs::path& dir, u32 version, + const std::vector<std::string>& skip, u8 flags = 0, u64 fallback_id = 0) { + const auto index = static_cast<u32>(plan.packages.size()); + PlanPackage package{kind, flags, fallback_id, version, {}}; + auto& tlvs = package_tlvs.emplace_back(); + + const auto exefs = FindDirectory(dir, "exefs"); + const auto romfs = FindDirectory(dir, "romfs"); + fs::path npdm_path = exefs.empty() ? fs::path{} : exefs / "main.npdm"; + if (!IsFile(npdm_path)) { + npdm_path = dir / "main.npdm"; + } + if (IsFile(npdm_path)) { + auto npdm = ReadWholeFile(npdm_path); + if (const auto id = NpdmProgramId(npdm); kind != KindAddon && id && *id != 0) { + package.title_id = *id; + } + tlvs.emplace_back(TlvNpdm, std::move(npdm)); + } + + if (kind != KindAddon) { + if (auto loader = OpenLoader(vfs, dir)) { + std::string display_version; + if (loader->ReadDisplayVersion(display_version) == Loader::ResultStatus::Success && + !display_version.empty()) { + tlvs.emplace_back(TlvDisplayVersion, StringBytes(display_version)); + } + loaders.emplace_back(index, std::move(loader)); + } + } + plan.packages.push_back(std::move(package)); + + if (!exefs.empty()) { + CollectTree(index, exefs, "exefs"); + } + if (!romfs.empty()) { + CollectTree(index, romfs, "romfs"); + } + std::error_code ec; + for (const auto& entry : fs::directory_iterator(dir, ec)) { + const auto name = ToUtf8(entry.path().filename()); + const auto lower = Lower(name); + if (entry.path() == exefs || entry.path() == romfs || + std::ranges::find(skip, lower) != skip.end()) { + continue; + } + if (entry.is_directory(ec)) { + CollectTree(index, entry.path(), "data/" + name); + } else { + AddFile(index, entry, "data/" + name); + } + } + return plan.packages.back().title_id; + } + + // Names, icon and control data belong to the base package, or the first one without a base. + // Our dumps rarely carry a control folder; when one does, the base's own wins, then updates. + void AddApplicationMetadata() { + const auto app = static_cast<u32>(std::distance( + plan.packages.begin(), std::ranges::find(plan.packages, KindBase, &PlanPackage::kind))); + const u32 target = app < plan.packages.size() ? app : 0; + std::ranges::stable_sort(loaders, [target](const auto& a, const auto& b) { + return (a.first == target) > (b.first == target); + }); + + auto& tlvs = package_tlvs[target]; + bool have_control = false; + bool have_icon = false; + for (const auto& [index, loader] : loaders) { + NACP control; + if (!have_control && + loader->ReadControlData(control) == Loader::ResultStatus::Success) { + have_control = true; + tlvs.emplace_back(TlvNacp, control.GetRawBytes()); + if (const auto name = control.GetApplicationName(); !name.empty()) { + tlvs.emplace_back(TlvTitleName, StringBytes(name)); + } + if (const auto publisher = control.GetDeveloperName(); !publisher.empty()) { + tlvs.emplace_back(TlvPublisher, StringBytes(publisher)); + } + } + std::vector<u8> icon; + if (!have_icon && loader->ReadIcon(icon) == Loader::ResultStatus::Success && + !icon.empty()) { + have_icon = true; + tlvs.emplace_back(TlvIcon, std::move(icon)); + } + } + } + + bool IsExcluded(const fs::path& path) const { + const auto normal = fs::absolute(path).lexically_normal(); + return std::ranges::find(excluded, normal) != excluded.end(); + } + + void AddFile(u32 package, const fs::directory_entry& entry, const std::string& path) { + std::error_code ec; + if (entry.is_symlink(ec)) { + throw PackError(PackResult::InvalidSource, + "symlinks are not supported: " + ToUtf8(entry.path())); + } + if (!entry.is_regular_file(ec)) { + throw PackError(PackResult::InvalidSource, + "unsupported file type: " + ToUtf8(entry.path())); + } + // Dump-tool listings are excluded by the spec. + if (entry.path().filename() == ".listing" || IsExcluded(entry.path())) { + return; + } + plan.entries.push_back({package, path, Relative(entry.path()), false}); + } + + void CollectTree(u32 package, const fs::path& root, const std::string& prefix) { + plan.entries.push_back({package, prefix, Relative(root), true}); + for (auto it = fs::recursive_directory_iterator(root); + it != fs::recursive_directory_iterator(); ++it) { + const auto& entry = *it; + const auto path = prefix + "/" + ToUtf8(fs::relative(entry.path(), root)); + if (!entry.is_symlink() && entry.is_directory()) { + plan.entries.push_back({package, path, Relative(entry.path()), true}); + } else { + AddFile(package, entry, path); + } + } + } + + std::string Relative(const fs::path& path) const { + return ToUtf8(fs::relative(path, dump_root)); + } + + fs::path dump_root; + std::vector<fs::path> excluded; + RealVfsFilesystem vfs; + Plan plan; + std::vector<std::vector<std::pair<u16, std::vector<u8>>>> package_tlvs; + std::vector<std::pair<u32, std::unique_ptr<Loader::AppLoader_DeconstructedRomDirectory>>> + loaders; +}; + +Json PlanToJson(const Plan& plan) { + Json packages = Json::array(); + for (const auto& package : plan.packages) { + packages.push_back(Json{{"kind", package.kind}, + {"flags", package.flags}, + {"title_id", package.title_id}, + {"version", package.version}, + {"meta", ToHex(package.meta)}}); + } + Json entries = Json::array(); + for (const auto& entry : plan.entries) { + entries.push_back(Json{{"package", entry.package}, + {"path", entry.path}, + {"source", entry.source}, + {"directory", entry.directory}}); + } + return Json{{"format", "nup-journal"}, + {"version", JournalVersion}, + {"packages", packages}, + {"entries", entries}}; +} + +Plan PlanFromJson(const Json& json) { + if (json.at("format") != "nup-journal" || json.at("version") != JournalVersion) { + throw PackError(PackResult::IoError, "unsupported journal"); + } + Plan plan; + for (const auto& package : json.at("packages")) { + plan.packages.push_back({package.at("kind").get<u8>(), package.at("flags").get<u8>(), + package.at("title_id").get<u64>(), + package.at("version").get<u32>(), + FromHex(package.at("meta").get<std::string>())}); + } + for (const auto& entry : json.at("entries")) { + plan.entries.push_back({entry.at("package").get<u32>(), entry.at("path").get<std::string>(), + entry.at("source").get<std::string>(), + entry.at("directory").get<bool>()}); + } + return plan; +} + +void RemoveFile(const fs::path& path) { + std::error_code ec; + fs::remove(path, ec); +} + +bool IsEmptyDirectory(const fs::path& path) { + std::error_code ec; + return fs::is_directory(path, ec) && fs::is_empty(path, ec) && !ec; +} + +void RemoveEmptyTree(const fs::path& root) { + std::error_code ec; + std::vector<fs::path> directories; + for (auto it = fs::recursive_directory_iterator(root, ec); + !ec && it != fs::recursive_directory_iterator(); it.increment(ec)) { + if (it->is_directory(ec)) { + directories.push_back(it->path()); + } + } + for (auto it = directories.rbegin(); it != directories.rend(); ++it) { + if (IsEmptyDirectory(*it)) { + fs::remove(*it, ec); + } + } + if (IsEmptyDirectory(root)) { + fs::remove(root, ec); + } +} + +void RemoveEmptyParents(fs::path path, const fs::path& dump_root) { + while (!path.empty() && path.has_relative_path()) { + std::error_code ec; + if (fs::equivalent(path, dump_root, ec) || ec || !IsEmptyDirectory(path)) { + break; + } + fs::remove(path, ec); + path = path.parent_path(); + } +} + +class Packer { +public: + Packer(const fs::path& dump_root_, const fs::path& archive_, const PackOptions& options_, + const std::function<void(const PackProgress&)>& progress_, + const std::atomic_bool* cancel_) + : dump_root{dump_root_}, archive{archive_}, part{WithSuffix(archive_, ".part")}, + journal{WithSuffix(archive_, ".journal")}, options{options_}, progress_cb{progress_}, + cancel{cancel_} {} + + PackResult Run() { + std::error_code ec; + if (fs::exists(archive, ec)) { + throw PackError(PackResult::IoError, "archive already exists: " + ToUtf8(archive)); + } + // A journal without a complete plan line was cut short before anything was deleted. + const bool resume = + options.delete_sources && HasResumableJournal(archive) && LoadJournal(); + if (resume) { + OpenForResume(); + } else { + if (!IsLooseDump(dump_root)) { + throw PackError(PackResult::InvalidSource, + "expected exefs/main, exefs/main.npdm and a romfs folder"); + } + std::vector<fs::path> excluded; + for (const auto& path : {archive, part, journal}) { + excluded.push_back(fs::absolute(path).lexically_normal()); + } + plan = Discovery{dump_root, std::move(excluded)}.Run(); + layout = BuildLayout(plan); + OpenNew(); + } + extents.resize(layout.files.size()); + done.resize(layout.files.size()); + for (const auto& [index, extent] : journaled) { + extents[index] = extent; + done[index] = true; + Remember(extent); + } + CountTotals(); + Report(); + + for (size_t i = 0; i < layout.files.size(); ++i) { + if (done[i]) { + continue; + } + PackFile(i); + if (options.delete_sources && + (chunk_bytes >= options.chunk_bytes || chunk.size() >= MaxChunkFiles)) { + Checkpoint(); + } + } + if (options.delete_sources) { + Checkpoint(); + } + + Finish(); + out.Close(); + fs::rename(part, archive, ec); + if (ec) { + throw PackError(PackResult::IoError, "cannot rename archive: " + ec.message()); + } + + if (options.delete_sources) { + RemoveFile(journal); + RemoveSourceFolders(); + } + return PackResult::Success; + } + + void Abort() { + out.Close(); + if (!options.delete_sources) { + RemoveFile(part); + } + } + +private: + void OpenNew() { + RemoveFile(journal); + out.Open(part, Common::FS::FileAccessMode::Write); + if (!out.IsOpen()) { + throw PackError(PackResult::IoError, "cannot create " + ToUtf8(part)); + } + WriteZeros(layout.blob_offset); + cursor = layout.blob_offset; + if (options.delete_sources) { + AppendJournal(PlanToJson(plan).dump() + '\n'); + } + } + + bool LoadJournal() { + const auto contents = + Common::FS::ReadStringFromFile(journal, Common::FS::FileType::BinaryFile); + size_t start = 0; + bool have_plan = false; + while (start < contents.size()) { + const auto line_end = contents.find('\n', start); + if (line_end == std::string::npos) { + break; + } + const auto line = std::string_view{contents}.substr(start, line_end - start); + start = line_end + 1; + Json json; + try { + json = Json::parse(line); + } catch (const Json::exception&) { + // Only the tail can be torn: the journal is synced before sources are deleted. + break; + } + if (!have_plan) { + plan = PlanFromJson(json); + layout = BuildLayout(plan); + resume_end = layout.blob_offset; + have_plan = true; + continue; + } + const auto index = json.at("file").get<size_t>(); + const auto sha = FromHex(json.at("sha256").get<std::string>()); + if (index >= layout.files.size() || sha.size() != 32) { + throw PackError(PackResult::IoError, "malformed journal"); + } + Extent extent{json.at("size").get<u64>(), {}, json.at("offset").get<u64>(), + json.at("stored_size").get<u64>(), json.at("codec").get<u8>()}; + std::ranges::copy(sha, extent.sha256.begin()); + resume_end = std::max(resume_end, extent.offset + extent.stored_size); + journaled.emplace_back(index, extent); + } + return have_plan; + } + + void OpenForResume() { + const u64 end = resume_end; + out.Open(part, Common::FS::FileAccessMode::ReadWrite); + if (!out.IsOpen()) { + throw PackError(PackResult::IoError, "cannot open " + ToUtf8(part)); + } + if (out.GetSize() < end) { + throw PackError(PackResult::IoError, "partial archive is shorter than its journal; " + "delete " + + ToUtf8(part) + " and " + ToUtf8(journal) + + " to start over"); + } + if (!out.SetSize(end) || !out.Seek(static_cast<s64>(end))) { + throw PackError(PackResult::IoError, "cannot truncate " + ToUtf8(part)); + } + cursor = end; + + // Journaled before an interruption, but the delete never happened. + for (const auto& [index, extent] : journaled) { + RemoveFile(SourceOf(index)); + } + } + + fs::path SourceOf(size_t file_index) const { + return dump_root / FromUtf8(plan.entries[layout.files[file_index].entry].source); + } + + void CountTotals() { + for (size_t i = 0; i < layout.files.size(); ++i) { + ++progress.files_total; + if (done[i]) { + progress.bytes_total += extents[i].size; + progress.bytes_done += extents[i].size; + ++progress.files_done; + } else { + std::error_code ec; + const auto size = fs::file_size(SourceOf(i), ec); + if (ec) { + throw PackError(PackResult::InvalidSource, + "missing source file: " + ToUtf8(SourceOf(i))); + } + progress.bytes_total += size; + } + } + } + + void Remember(const Extent& extent) { + if (extent.size == 0) { + return; + } + sizes_seen.insert(extent.size); + by_hash.try_emplace(extent.sha256, extent); + } + + bool Cancelled() const { + return cancel && cancel->load(); + } + + void WriteZeros(u64 count) { + static constexpr std::array<u8, 4096> zeros{}; + while (count > 0) { + const auto n = static_cast<size_t>(std::min<u64>(count, zeros.size())); + Write(zeros.data(), n); + count -= n; + } + } + + void Write(const u8* data, size_t size) { + if (out.WriteSpan(std::span<const u8>{data, size}) != size) { + throw PackError(PackResult::IoError, "write failed, the disk may be full"); + } + } + + void AddProgress(u64 bytes) { + progress.bytes_done += bytes; + if (progress.bytes_done - last_report >= ProgressStep) { + Report(); + } + } + + // Reads the whole source through `sink`, checking that it still has the expected size. A file + // read more than once (hashed for dedup, or compressed and then stored raw) counts once. + template <typename Sink> + void ReadSource(Common::FS::IOFile& in, u64 size, Sink&& sink) { + if (!in.Seek(0)) { + throw PackError(PackResult::IoError, "cannot seek source"); + } + buffer.resize(ReadBufferSize); + u64 total = 0; + while (true) { + if (Cancelled()) { + throw PackError(PackResult::Cancelled, "cancelled"); + } + const auto read = in.ReadSpan(std::span{buffer}); + if (read == 0) { + break; + } + total += read; + if (total > size) { + break; + } + if (total > file_counted) { + AddProgress(total - file_counted); + file_counted = total; + } + if (!sink(buffer.data(), read)) { + return; + } + } + if (total != size) { + throw PackError(PackResult::IoError, "a source file changed while packing"); + } + } + + Digest HashSource(Common::FS::IOFile& in, u64 size) { + hash.Reset(); + ReadSource(in, size, [this](const u8* data, size_t n) { + hash.Update(data, n); + return true; + }); + return hash.Final(); + } + + ZstdContext& Compressor() { + if (!compressor) { + compressor = std::make_unique<ZstdContext>(options.zstd_level); + } + return *compressor; + } + + // Streams one zstd frame into the archive. Gives up (returns false) as soon as the output is + // no smaller than the input. + bool WriteCompressed(Common::FS::IOFile& in, u64 size, u64& stored) { + auto& zstd = Compressor(); + zstd.Begin(size); + out_buffer.resize(ZSTD_CStreamOutSize()); + stored = 0; + bool fits = true; + const auto drain = [&](ZSTD_inBuffer& input, ZSTD_EndDirective mode) { + while (true) { + ZSTD_outBuffer output{out_buffer.data(), out_buffer.size(), 0}; + const size_t remaining = ZSTD_compressStream2(zstd.get(), &output, &input, mode); + if (ZSTD_isError(remaining)) { + throw PackError(PackResult::IoError, + std::string{"zstd: "} + ZSTD_getErrorName(remaining)); + } + stored += output.pos; + if (stored >= size) { + return false; + } + Write(out_buffer.data(), output.pos); + const bool finished = mode == ZSTD_e_end ? remaining == 0 + : input.pos == input.size; + if (finished) { + return true; + } + } + }; + hash.Reset(); + ReadSource(in, size, [&](const u8* data, size_t n) { + hash.Update(data, n); + ZSTD_inBuffer input{data, n, 0}; + fits = drain(input, ZSTD_e_continue); + return fits; + }); + if (!fits) { + return false; + } + ZSTD_inBuffer empty{nullptr, 0, 0}; + return drain(empty, ZSTD_e_end); + } + + void PackFile(size_t index) { + const auto source = SourceOf(index); + Common::FS::IOFile in{source, Common::FS::FileAccessMode::Read}; + if (!in.IsOpen()) { + throw PackError(PackResult::IoError, "cannot read " + ToUtf8(source)); + } + Extent extent; + extent.size = in.GetSize(); + file_counted = 0; + + std::optional<Digest> known_hash; + if (extent.size == 0) { + known_hash = HashSource(in, 0); + } else if (sizes_seen.contains(extent.size)) { + // Only a file whose size matches an earlier extent can be a duplicate; hash it before + // writing anything so the shared extent costs no output. + known_hash = HashSource(in, extent.size); + if (const auto it = by_hash.find(*known_hash); + it != by_hash.end() && it->second.size == extent.size) { + extent = it->second; + Complete(index, extent); + return; + } + } + + if (extent.size == 0) { + extent.sha256 = *known_hash; + Complete(index, extent); + return; + } + + const u64 start = AlignUp(cursor, ExtentAlignment); + Seek(cursor); + WriteZeros(start - cursor); + extent.offset = start; + + bool compressed = false; + if (extent.size <= options.raw_threshold && options.zstd_level > 0) { + u64 stored = 0; + compressed = WriteCompressed(in, extent.size, stored); + if (compressed) { + extent.codec = CodecZstd; + extent.stored_size = stored; + extent.sha256 = hash.Final(); + } + } + if (!compressed) { + Seek(start); + hash.Reset(); + ReadSource(in, extent.size, [this](const u8* data, size_t n) { + hash.Update(data, n); + Write(data, n); + return true; + }); + extent.codec = CodecRaw; + extent.stored_size = extent.size; + extent.sha256 = hash.Final(); + } + cursor = start + extent.stored_size; + Remember(extent); + Complete(index, extent); + } + + void Seek(u64 offset) { + if (!out.Seek(static_cast<s64>(offset))) { + throw PackError(PackResult::IoError, "cannot seek " + ToUtf8(part)); + } + } + + void Complete(size_t index, const Extent& extent) { + extents[index] = extent; + done[index] = true; + if (options.delete_sources) { + chunk.push_back(index); + chunk_bytes += extent.size; + } + ++progress.files_done; + Report(); + } + + void AppendJournal(const std::string& lines) { + Common::FS::IOFile log{journal, Common::FS::FileAccessMode::Append}; + if (!log.IsOpen() || log.WriteString(lines) != lines.size() || !log.Commit()) { + throw PackError(PackResult::IoError, "cannot write " + ToUtf8(journal)); + } + } + + // Archive data first, then the journal, then the deletes, so an interruption loses nothing. + void Checkpoint() { + if (chunk.empty()) { + return; + } + if (!out.Commit()) { + throw PackError(PackResult::IoError, "cannot sync " + ToUtf8(part)); + } + std::string lines; + for (const auto index : chunk) { + const auto& extent = extents[index]; + lines += Json{{"file", index}, + {"size", extent.size}, + {"sha256", ToHex(extent.sha256)}, + {"offset", extent.offset}, + {"stored_size", extent.stored_size}, + {"codec", extent.codec}} + .dump(); + lines += '\n'; + } + AppendJournal(lines); + for (const auto index : chunk) { + RemoveFile(SourceOf(index)); + } + chunk.clear(); + chunk_bytes = 0; + } + + void Finish() { + // Meta blocks follow the last extent, as pack.py writes them. + Seek(cursor); + std::vector<std::pair<u64, u32>> meta; + for (const auto& package : plan.packages) { + if (package.meta.size() > std::numeric_limits<u32>::max()) { + throw PackError(PackResult::IoError, "package metadata is too large"); + } + meta.emplace_back(package.meta.empty() ? 0 : cursor, + static_cast<u32>(package.meta.size())); + Write(package.meta.data(), package.meta.size()); + cursor += package.meta.size(); + } + const u64 file_size = cursor; + + const auto toc = BuildToc(meta); + std::array<u8, 32> toc_hash{}; + unsigned int hash_length{}; + EVP_Digest(toc.data(), toc.size(), toc_hash.data(), &hash_length, EVP_sha256(), nullptr); + + // UUID: SHA-256 over title_id (u64), version (u32) and kind (u8) of each package. + Sha256 uuid_hash; + for (const auto& package : plan.packages) { + std::vector<u8> identity(13); + PutLE(identity, 0, package.title_id, 8); + PutLE(identity, 8, package.version, 4); + identity[12] = package.kind; + uuid_hash.Update(identity.data(), identity.size()); + } + const auto uuid = uuid_hash.Final(); + + std::vector<u8> super(SuperblockSize); + PutBytes(super, 0x00, Magic); + PutLE(super, 0x04, SuperblockSize, 2); + PutLE(super, 0x06, 1, 2); + PutLE(super, 0x08, SuperblockSize, 8); + PutLE(super, 0x10, toc.size(), 8); + PutBytes(super, 0x18, toc_hash); + PutLE(super, 0x38, layout.blob_offset, 8); + PutLE(super, 0x40, file_size, 8); + PutLE(super, 0x48, plan.packages.size(), 4); + PutBytes(super, 0x50, std::span{uuid}.first(16)); + + Seek(0); + Write(super.data(), super.size()); + Write(toc.data(), toc.size()); + if (!out.SetSize(file_size) || !out.Commit()) { + throw PackError(PackResult::IoError, "cannot finish " + ToUtf8(part)); + } + } + + // Every field is written little-endian at its documented offset. The package record is + // 0x40 bytes with 20 reserved bytes at 0x2C, as NupContainer reads it. + std::vector<u8> BuildToc(const std::vector<std::pair<u64, u32>>& meta) const { + std::vector<u8> toc(layout.toc_size); + const u64 dir_count = layout.dirs.size(); + PutBytes(toc, 0x00, TocMagic); + PutLE(toc, 0x04, plan.packages.size(), 4); + PutLE(toc, 0x08, dir_count, 8); + PutLE(toc, 0x10, layout.files.size(), 8); + + size_t pos = TocHeaderSize; + for (size_t i = 0; i < plan.packages.size(); ++i) { + const auto& package = plan.packages[i]; + const auto& record = layout.packages[i]; + PutLE(toc, pos + 0x00, package.title_id, 8); + PutLE(toc, pos + 0x08, record.root_dir, 4); + PutLE(toc, pos + 0x0C, record.exefs_dir, 4); + PutLE(toc, pos + 0x10, record.romfs_dir, 4); + PutLE(toc, pos + 0x14, package.version, 4); + toc[pos + 0x18] = package.kind; + toc[pos + 0x19] = package.flags; + PutLE(toc, pos + 0x20, meta[i].first, 8); + PutLE(toc, pos + 0x28, meta[i].second, 4); + pos += PackageRecordSize; + } + for (const auto& dir : layout.dirs) { + PutLE(toc, pos + 0x00, layout.pool_index.at(dir.name), 4); + PutLE(toc, pos + 0x04, dir.name.size(), 4); + PutLE(toc, pos + 0x08, dir.first_child, 4); + PutLE(toc, pos + 0x0C, dir.child_count, 4); + pos += DirRecordSize; + } + for (size_t i = 0; i < layout.files.size(); ++i) { + const auto& file = layout.files[i]; + const auto& extent = extents[i]; + PutLE(toc, pos + 0x00, layout.pool_index.at(file.name), 4); + PutLE(toc, pos + 0x04, file.name.size(), 4); + PutLE(toc, pos + 0x08, extent.size, 8); + // Empty files have no extent: offset, size and codec are all zero. + if (extent.size != 0) { + PutLE(toc, pos + 0x10, extent.offset, 8); + PutLE(toc, pos + 0x18, extent.stored_size, 8); + toc[pos + 0x40] = extent.codec; + } + PutBytes(toc, pos + 0x20, extent.sha256); + pos += FileRecordSize; + } + for (const u64 child : layout.children) { + const u64 id = (child & Layout::FileTag) ? dir_count + static_cast<u32>(child) : child; + PutLE(toc, pos, id, 4); + pos += sizeof(u32); + } + PutLE(toc, 0x18, pos, 8); + PutLE(toc, 0x20, layout.pool.size(), 8); + std::memcpy(toc.data() + pos, layout.pool.data(), layout.pool.size()); + return toc; + } + + void RemoveSourceFolders() { + for (const auto& entry : plan.entries) { + const auto depth = std::ranges::count(entry.path, '/'); + const bool top = entry.directory && + (depth == 0 || (depth == 1 && entry.path.starts_with("data/"))); + if (top) { + const auto source = dump_root / FromUtf8(entry.source); + RemoveEmptyTree(source); + RemoveEmptyParents(source.parent_path(), dump_root); + } + } + } + + void Report() { + last_report = progress.bytes_done; + if (progress_cb) { + progress_cb(progress); + } + } + + fs::path dump_root; + fs::path archive; + fs::path part; + fs::path journal; + PackOptions options; + std::function<void(const PackProgress&)> progress_cb; + const std::atomic_bool* cancel; + + Plan plan; + Layout layout; + std::vector<Extent> extents; + std::vector<bool> done; + std::vector<std::pair<size_t, Extent>> journaled; + + struct DigestHash { + size_t operator()(const Digest& digest) const { + size_t value; + std::memcpy(&value, digest.data(), sizeof(value)); + return value; + } + }; + std::unordered_map<Digest, Extent, DigestHash> by_hash; + std::unordered_set<u64> sizes_seen; + + Common::FS::IOFile out; + u64 cursor = 0; + u64 resume_end = 0; + std::vector<size_t> chunk; + u64 chunk_bytes = 0; + std::vector<u8> buffer; + std::vector<u8> out_buffer; + Sha256 hash; + std::unique_ptr<ZstdContext> compressor; + PackProgress progress{}; + u64 last_report = 0; + u64 file_counted = 0; +}; + +} // namespace + +bool IsLooseDump(const fs::path& dir) { + return HasBootableExeFS(dir); +} + +std::optional<fs::path> DumpRootFromMain(const fs::path& main) { + // The Qt game list stores the exefs folder rather than exefs/main. + std::error_code ec; + const auto exefs = fs::is_directory(main, ec) ? main : main.parent_path(); + if (Lower(ToUtf8(exefs.filename())) != "exefs") { + return std::nullopt; + } + auto root = exefs.parent_path(); + const auto name = Lower(ToUtf8(root.filename())); + if (name == "base") { + root = root.parent_path(); + } else if (ParseDecimal(name) && Lower(ToUtf8(root.parent_path().filename())) == "updates") { + root = root.parent_path().parent_path(); + } + if (!IsLooseDump(root)) { + return std::nullopt; + } + return root; +} + +fs::path DefaultArchivePath(const fs::path& dump_root) { + auto root = dump_root; + if (!root.has_filename()) { + root = root.parent_path(); + } + return root.parent_path() / (root.filename().u8string() + u8".nup"); +} + +bool HasResumableJournal(const fs::path& archive) { + std::error_code ec; + return fs::exists(WithSuffix(archive, ".part"), ec) && + fs::exists(WithSuffix(archive, ".journal"), ec); +} + +PackResult PackDump(const fs::path& dump_root, const fs::path& archive, const PackOptions& options, + const std::function<void(const PackProgress&)>& progress, + const std::atomic_bool* cancel, std::string* error_out) { + Packer packer{dump_root, archive, options, progress, cancel}; + try { + return packer.Run(); + } catch (const PackError& e) { + packer.Abort(); + if (error_out) { + *error_out = e.what(); + } + return e.result; + } catch (const std::exception& e) { + packer.Abort(); + if (error_out) { + *error_out = e.what(); + } + return PackResult::IoError; + } +} + +} // namespace FileSys::Nup diff --git a/src/core/file_sys/nup_writer.h b/src/core/file_sys/nup_writer.h new file mode 100644 index 0000000000..231387b156 --- /dev/null +++ b/src/core/file_sys/nup_writer.h @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <atomic> +#include <filesystem> +#include <functional> +#include <optional> +#include <string> + +#include "common/common_types.h" + +// Packs a loose dump into a NUP container. Every package is kept: base/, each +// updates/<version>/ and each dlc/<titleid>/<n>/. +namespace FileSys::Nup { + +struct PackProgress { + u64 bytes_done{}; + u64 bytes_total{}; + u64 files_done{}; + u64 files_total{}; +}; + +enum class PackResult { Success, Cancelled, InvalidSource, IoError }; + +// NupContainer keeps a zstd file in memory only while it fits its 512 MiB decompression cache; +// a larger one is decompressed again in full on every read, which makes big compressed RomFS +// files unusable. Files above this size are therefore stored raw, with headroom below the cap so +// that one file does not evict everything else from the cache. +constexpr u64 DefaultRawThreshold = 256ULL * 1024 * 1024; + +struct PackOptions { + // Low-storage mode: delete each source file once its extent is synced and journaled. + bool delete_sources = false; + u64 chunk_bytes = 256ULL * 1024 * 1024; + int zstd_level = 3; + // Files larger than this are always stored raw. + u64 raw_threshold = DefaultRawThreshold; +}; + +// A loose dump holds exefs/main and main.npdm, directly or under base/ or updates/<version>/, +// and a romfs folder. Folder names are matched case-insensitively. +bool IsLooseDump(const std::filesystem::path& dir); +// The dump folder a game list entry's exefs/main (or exefs folder) belongs to, if it is a loose +// dump. +std::optional<std::filesystem::path> DumpRootFromMain(const std::filesystem::path& main); +std::filesystem::path DefaultArchivePath(const std::filesystem::path& dump_root); + +// Writes <archive>.part and renames it when done; an existing archive is never overwritten. A +// cancelled or failed low-storage run keeps its .part and .journal, and calling this again with +// the same arguments resumes it. +PackResult PackDump(const std::filesystem::path& dump_root, const std::filesystem::path& archive, + const PackOptions& options = {}, + const std::function<void(const PackProgress&)>& progress = {}, + const std::atomic_bool* cancel = nullptr, std::string* error_out = nullptr); + +bool HasResumableJournal(const std::filesystem::path& archive); + +} // namespace FileSys::Nup diff --git a/src/core/file_sys/system_archive/system_archive.cpp b/src/core/file_sys/system_archive/system_archive.cpp index 1743d01b12..c3931b6f62 100644 --- a/src/core/file_sys/system_archive/system_archive.cpp +++ b/src/core/file_sys/system_archive/system_archive.cpp @@ -4,6 +4,8 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include <cstdlib> + #include "common/logging.h" #include "core/file_sys/romfs.h" #include "core/file_sys/system_archive/mii_model.h" @@ -12,6 +14,7 @@ #include "core/file_sys/system_archive/system_archive.h" #include "core/file_sys/system_archive/system_version.h" #include "core/file_sys/system_archive/time_zone_binary.h" +#include "core/file_sys/vfs/vfs_vector.h" namespace FileSys::SystemArchive { @@ -76,6 +79,22 @@ constexpr inline SystemArchiveDescriptor GetSystemArchive(u64 title_id) { VirtualFile SynthesizeSystemArchive(const u64 title_id) { auto const desc = GetSystemArchive(title_id); + if (desc.name == nullptr) { + // Not a system archive Eden knows (fmt throws on a null string, ending the process). + // Switch 2 system data (ids 04...) is not installed with Switch 1 firmware, and titles + // abort when it cannot be mounted (Nintendo Switch Sports Resort, 0400000000000849): + // serve an empty archive. EDEN_NX2_EMPTY_SYSTEM_DATA=0 fails the request instead. + static const bool empty_nx2 = [] { + const char* setting = std::getenv("EDEN_NX2_EMPTY_SYSTEM_DATA"); + return setting == nullptr || setting[0] != '0'; + }(); + if (empty_nx2 && (title_id >> 56) == 0x04) { + LOG_WARNING(Service_FS, "Empty archive for Switch 2 system data {:#016x}", title_id); + return CreateRomFS(std::make_shared<VectorVfsDirectory>()); + } + LOG_WARNING(Service_FS, "No system archive {:#016x} to synthesize", title_id); + return nullptr; + } LOG_INFO(Service_FS, "Synthesizing system archive '{}' ({:#016x}).", desc.name, title_id); if (desc.supplier != nullptr) { if (auto const dir = desc.supplier(); dir != nullptr) { diff --git a/src/core/file_sys/vfs/vfs_real.cpp b/src/core/file_sys/vfs/vfs_real.cpp index 5b5b54017c..7cc5315a70 100644 --- a/src/core/file_sys/vfs/vfs_real.cpp +++ b/src/core/file_sys/vfs/vfs_real.cpp @@ -114,7 +114,6 @@ VfsEntryType RealVfsFilesystem::GetEntryType(std::string_view path_) const { } VirtualFile RealVfsFilesystem::OpenFileFromEntry(std::string_view path_, std::optional<u64> size, - std::optional<std::string> parent_path, OpenMode perms) { const auto path = FS::SanitizePath(path_, FS::DirectorySeparator::PlatformDefault); std::scoped_lock lk{list_lock}; @@ -132,14 +131,14 @@ VirtualFile RealVfsFilesystem::OpenFileFromEntry(std::string_view path_, std::op auto reference = std::make_unique<FileReference>(); this->InsertReferenceIntoListLocked(*reference); - auto file = std::make_shared<RealVfsFile>(*this, std::move(reference), path, perms, size, std::move(parent_path)); + auto file = std::make_shared<RealVfsFile>(*this, std::move(reference), path, perms, size); cache[path] = file; return file; } VirtualFile RealVfsFilesystem::OpenFile(std::string_view path_, OpenMode perms) { - return OpenFileFromEntry(path_, {}, {}, perms); + return OpenFileFromEntry(path_, {}, perms); } VirtualFile RealVfsFilesystem::CreateFile(std::string_view path_, OpenMode perms) { @@ -257,7 +256,8 @@ std::unique_lock<std::mutex> RealVfsFilesystem::RefreshReference(const std::stri return lk; } -void RealVfsFilesystem::DropReference(std::unique_ptr<FileReference>&& reference) { +void RealVfsFilesystem::DropReference(std::unique_ptr<FileReference>&& reference, + const std::string& path) { if (in_dtor) return; @@ -266,6 +266,11 @@ void RealVfsFilesystem::DropReference(std::unique_ptr<FileReference>&& reference // Remove from list. this->RemoveReferenceFromListLocked(*reference); + // Forget the path, unless it was opened again since this file's last reference went away. + if (const auto it = cache.find(path); it != cache.end() && it->second.expired()) { + cache.erase(it); + } + // Close the file. if (reference->file) { reference->file.reset(); @@ -309,14 +314,11 @@ void RealVfsFilesystem::RemoveReferenceFromListLocked(FileReference& reference) } RealVfsFile::RealVfsFile(RealVfsFilesystem& base_, std::unique_ptr<FileReference> reference_, - const std::string& path_, OpenMode perms_, std::optional<u64> size_, - std::optional<std::string> parent_path_) - : base(base_), reference(std::move(reference_)), path(path_), - parent_path(parent_path_ ? std::move(*parent_path_) : FS::GetParentPath(path_)), - path_components(FS::SplitPathComponentsCopy(path_)), size(size_), perms(perms_) {} + const std::string& path_, OpenMode perms_, std::optional<u64> size_) + : base(base_), reference(std::move(reference_)), path(path_), size(size_), perms(perms_) {} RealVfsFile::~RealVfsFile() { - base.DropReference(std::move(reference)); + base.DropReference(std::move(reference), path); } std::string RealVfsFile::GetName() const { @@ -325,7 +327,14 @@ std::string RealVfsFile::GetName() const { return FS::Android::GetFilename(path); } #endif - return path_components.empty() ? "" : std::string(path_components.back()); + // The last path component. + const auto end = path.find_last_not_of("\\/"); + if (end == std::string::npos) { + return ""; + } + const auto separator = path.find_last_of("\\/", end); + const auto begin = separator == std::string::npos ? 0 : separator + 1; + return path.substr(begin, end + 1 - begin); } std::size_t RealVfsFile::GetSize() const { @@ -343,7 +352,7 @@ bool RealVfsFile::Resize(std::size_t new_size) { } VirtualDir RealVfsFile::GetContainingDirectory() const { - return base.OpenDirectory(parent_path, perms); + return base.OpenDirectory(FS::GetParentPath(path), perms); } bool RealVfsFile::IsWritable() const { @@ -372,7 +381,7 @@ std::size_t RealVfsFile::Write(const u8* data, std::size_t length, std::size_t o } bool RealVfsFile::Rename(std::string_view name) { - return base.MoveFile(path, parent_path + '/' + std::string(name)) != nullptr; + return base.MoveFile(path, FS::GetParentPath(path) + '/' + std::string(name)) != nullptr; } // TODO(DarkLordZach): MSVC would not let me combine the following two functions using 'if @@ -390,7 +399,7 @@ std::vector<VirtualFile> RealVfsDirectory::IterateEntries<RealVfsFile, VfsFile>( &out](const std::filesystem::directory_entry& entry) { const auto full_path_string = FS::PathToUTF8String(entry.path()); - out.emplace_back(base.OpenFileFromEntry(full_path_string, entry.file_size(), path, perms)); + out.emplace_back(base.OpenFileFromEntry(full_path_string, entry.file_size(), perms)); return true; }; diff --git a/src/core/file_sys/vfs/vfs_real.h b/src/core/file_sys/vfs/vfs_real.h index a97f7d32ea..2df125b7eb 100644 --- a/src/core/file_sys/vfs/vfs_real.h +++ b/src/core/file_sys/vfs/vfs_real.h @@ -62,12 +62,11 @@ private: friend class RealVfsFile; std::unique_lock<std::mutex> RefreshReference(const std::string& path, OpenMode perms, FileReference& reference); - void DropReference(std::unique_ptr<FileReference>&& reference); + void DropReference(std::unique_ptr<FileReference>&& reference, const std::string& path); private: friend class RealVfsDirectory; VirtualFile OpenFileFromEntry(std::string_view path, std::optional<u64> size, - std::optional<std::string> parent_path, OpenMode perms = OpenMode::Read); private: @@ -84,7 +83,7 @@ class RealVfsFile : public VfsFile { public: RealVfsFile(RealVfsFilesystem& base, std::unique_ptr<FileReference> reference, const std::string& path, OpenMode perms = OpenMode::Read, - std::optional<u64> size = {}, std::optional<std::string> parent_path = {}); + std::optional<u64> size = {}); ~RealVfsFile() override; std::string GetName() const override; @@ -101,9 +100,8 @@ private: RealVfsFilesystem& base; std::unique_ptr<FileReference> reference; + // Only the path: a loose RomFS keeps one of these per file (title 1: 206k). std::string path; - std::string parent_path; - std::vector<std::string> path_components; std::optional<u64> size; OpenMode perms; }; diff --git a/src/core/hle/kernel/board/nintendo/nx/k_system_control.h b/src/core/hle/kernel/board/nintendo/nx/k_system_control.h index cc5572bde8..9aa527adb1 100644 --- a/src/core/hle/kernel/board/nintendo/nx/k_system_control.h +++ b/src/core/hle/kernel/board/nintendo/nx/k_system_control.h @@ -23,7 +23,10 @@ public: // Extra DRAM on the 12 GB layout for memory the Switch 2 graphics server keeps itself (nx2:gpu // 214 GPU-only regions). It sits in the non-secure system pool, outside the application pool. - static constexpr size_t Nx2GraphicsServerMemorySize = 8ULL * 1024 * 1024 * 1024; // 8_GiB + // GPU-only regions get memory only where guest memory is written and sparse regions take + // their tiles from the game's own memory (216), so much less of it is used (title 1: ~1.1 GiB, + // all of it at boot); all of it counts against the host's commit limit from boot. + static constexpr size_t Nx2GraphicsServerMemorySize = 2ULL * 1024 * 1024 * 1024; // 2_GiB public: class Init { diff --git a/src/core/hle/kernel/k_memory_manager.cpp b/src/core/hle/kernel/k_memory_manager.cpp index 743e290541..bed3dd819d 100644 --- a/src/core/hle/kernel/k_memory_manager.cpp +++ b/src/core/hle/kernel/k_memory_manager.cpp @@ -48,6 +48,8 @@ KMemoryManager::KMemoryManager(Core::System& system) } {} void KMemoryManager::Initialize(KVirtualAddress management_region, size_t management_region_size) { + // Memory a previous kernel handed out may still hold its data. + m_system.DeviceMemory().buffer.ForgetContents(); // Clear the management region to zero. const KVirtualAddress management_region_end = management_region + management_region_size; @@ -242,10 +244,21 @@ KPhysicalAddress KMemoryManager::AllocateAndOpenContinuous(size_t num_pages, siz // Open the first reference to the pages. chosen_manager->OpenFirst(allocated_block, num_pages); + this->MarkUsed(allocated_block, num_pages); return allocated_block; } +void KMemoryManager::MarkUsed(KPhysicalAddress address, size_t num_pages) { + m_system.DeviceMemory().buffer.MarkUsed(GetInteger(address) - Core::DramMemoryMap::Base, + num_pages * PageSize); +} + +void KMemoryManager::MarkDirty(KPhysicalAddress address, size_t num_pages) { + m_system.DeviceMemory().buffer.MarkDirty(GetInteger(address) - Core::DramMemoryMap::Base, + num_pages * PageSize); +} + Result KMemoryManager::AllocatePageGroupImpl(KPageGroup* out, size_t num_pages, Pool pool, Direction dir, bool unoptimized, bool random) { // Choose a heap based on our page size request. @@ -297,6 +310,10 @@ Result KMemoryManager::AllocatePageGroupImpl(KPageGroup* out, size_t num_pages, // Only succeed if we allocated as many pages as we wanted. R_UNLESS(num_pages == 0, ResultOutOfMemory); + for (const auto& block : *out) { + this->MarkUsed(block.GetAddress(), block.GetNumPages()); + } + // We succeeded! R_SUCCEED(); } @@ -525,9 +542,10 @@ bool KMemoryManager::Impl::ProcessOptimizedAllocation(KernelCore& kernel, KPhysi // If not, it's new. any_new = true; - // Fill the page. - auto* ptr = device_memory.GetPointer<u8>(m_heap.GetAddress()); - std::memset(ptr + offset * PageSize, fill_pattern, PageSize); + // Fill the page (a zero fill skips pages that are still zero). + device_memory.buffer.ClearBackingRegion( + GetInteger(m_heap.GetAddress()) + offset * PageSize - Core::DramMemoryMap::Base, + PageSize, fill_pattern); } offset++; diff --git a/src/core/hle/kernel/k_memory_manager.h b/src/core/hle/kernel/k_memory_manager.h index 2f4ae7f04f..66b97fef8a 100644 --- a/src/core/hle/kernel/k_memory_manager.h +++ b/src/core/hle/kernel/k_memory_manager.h @@ -100,6 +100,9 @@ public: } void Close(KPhysicalAddress address, size_t num_pages) { + // Pages that may be freed here may hold data: the next allocation clears them. + this->MarkDirty(address, num_pages); + // Repeatedly close references until we've done so for all pages. while (num_pages) { auto& manager = this->GetManager(address); @@ -333,6 +336,9 @@ private: }; private: + void MarkUsed(KPhysicalAddress address, size_t num_pages); + void MarkDirty(KPhysicalAddress address, size_t num_pages); + Impl& GetManager(KPhysicalAddress address) { return m_managers[m_memory_layout.GetPhysicalLinearRegion(address).GetAttributes()]; } diff --git a/src/core/hle/kernel/k_shared_memory.cpp b/src/core/hle/kernel/k_shared_memory.cpp index 61f2872560..cb8877b8b2 100644 --- a/src/core/hle/kernel/k_shared_memory.cpp +++ b/src/core/hle/kernel/k_shared_memory.cpp @@ -59,9 +59,10 @@ Result KSharedMemory::Initialize(KernelCore& kernel, Core::DeviceMemory& device_ // Mark initialized. m_is_initialized = true; - // Clear all pages in the memory. + // Clear all pages in the memory (pages still zero are skipped). for (const auto& block : *m_page_group) { - std::memset(m_device_memory->GetPointer<void>(block.GetAddress()), 0, block.GetSize()); + m_device_memory->buffer.ClearBackingRegion( + GetInteger(block.GetAddress()) - Core::DramMemoryMap::Base, block.GetSize(), 0); } R_SUCCEED(); diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp index 53d964ecbc..925eafc472 100644 --- a/src/core/hle/kernel/svc.cpp +++ b/src/core/hle/kernel/svc.cpp @@ -14,6 +14,8 @@ #include "core/core.h" #include "core/hle/kernel/k_process.h" #include "core/hle/kernel/svc.h" +#include "core/hle/kernel/k_readable_event.h" +#include "core/hle/service/kernel_helpers.h" namespace Kernel::Svc { @@ -2522,6 +2524,21 @@ void Call(Core::System& system, u32 imm) { LOG_INFO(Kernel_SVC, "svc {:#04x} in t{} [0]={:#x} [1]={:#x} [2]={:#x} [3]={:#x}", imm, GetCurrentThread(kernel).GetThreadId(), GetArg64(args, 0), GetArg64(args, 1), GetArg64(args, 2), GetArg64(args, 3)); + if (imm == 0x18) { + // WaitSynchronization: the handles, with the names services gave their events. + std::string list; + const s32 count = static_cast<s32>(GetArg64(args, 2)); + for (s32 i = 0; i < std::min(count, 8); ++i) { + const u32 handle = process.GetMemory().Read32(GetArg64(args, 1) + 4 * i); + KScopedAutoObject event = + process.GetHandleTable().GetObject<KReadableEvent>(kernel, handle); + list += fmt::format(" {:#x}{}", handle, + event.IsNotNull() + ? "=" + Service::KernelHelpers::DebugEventName(event.GetPointerUnsafe()) + : std::string{}); + } + LOG_INFO(Kernel_SVC, "svc 0x18 t{} waits on{}", GetCurrentThread(kernel).GetThreadId(), list); + } } if (imm == 0x80) { // Switch 2 only: svcSetGpuMemoryAttribute(address, size, mask, attr). nn::gpu::sdk and diff --git a/src/core/hle/service/acc/acc.cpp b/src/core/hle/service/acc/acc.cpp index d68b6bde0d..772da97ef6 100644 --- a/src/core/hle/service/acc/acc.cpp +++ b/src/core/hle/service/acc/acc.cpp @@ -696,9 +696,9 @@ public: {4, &IManagerForApplication::LoadIdTokenCache, "LoadIdTokenCache"}, {130, &IManagerForApplication::GetNintendoAccountUserResourceCacheForApplication, "GetNintendoAccountUserResourceCacheForApplication"}, {136, &IManagerForApplication::GetNintendoAccountUserResourceCacheForApplication, "GetNintendoAccountUserResourceCache"}, // 19.0.0+ - {150, nullptr, "CreateAuthorizationRequest"}, + {150, &IManagerForApplication::StubReplyInterface, "CreateAuthorizationRequest"}, {160, &IManagerForApplication::StoreOpenContext, "StoreOpenContext"}, - {170, nullptr, "LoadNetworkServiceLicenseKindAsync"}, + {170, &IManagerForApplication::StubReplyInterface, "LoadNetworkServiceLicenseKindAsync"}, }; // clang-format on diff --git a/src/core/hle/service/am/frontend/applet_software_keyboard.cpp b/src/core/hle/service/am/frontend/applet_software_keyboard.cpp index db42781e18..061394bff8 100644 --- a/src/core/hle/service/am/frontend/applet_software_keyboard.cpp +++ b/src/core/hle/service/am/frontend/applet_software_keyboard.cpp @@ -222,6 +222,14 @@ void SoftwareKeyboard::InitializeForeground() { sizeof(SwkbdConfigNew)); swkbd_config_common.use_utf8 = true; break; + case SwkbdAppletVersion::Version1441810: { + // Fire Emblem: Fortune's Weave asks for UTF-8 and keeps the reply up to the first NUL, so the + // Switch 1 layout's use_utf8 (a byte of the native initial_string_offset) left one letter. + const bool decoded = + DecodeNativeSwkbdConfig(swkbd_config_data, swkbd_config_common, swkbd_config_new); + ASSERT(decoded); + break; + } default: UNIMPLEMENTED_MSG("Unknown SwkbdConfig revision={} with size={}", swkbd_applet_version, swkbd_config_data.size()); diff --git a/src/core/hle/service/am/frontend/applet_software_keyboard_types.h b/src/core/hle/service/am/frontend/applet_software_keyboard_types.h index f263a704cd..8ff8391aa1 100644 --- a/src/core/hle/service/am/frontend/applet_software_keyboard_types.h +++ b/src/core/hle/service/am/frontend/applet_software_keyboard_types.h @@ -7,6 +7,9 @@ #pragma once #include <array> +#include <cstddef> +#include <cstring> +#include <span> #include "common/bit_field.h" #include "common/common_funcs.h" @@ -31,6 +34,7 @@ enum class SwkbdAppletVersion : u32_le { Version393227 = 0x6000B, // 6.0.0 - 7.0.1 Version524301 = 0x8000D, // 8.0.0+ Version1310736 = 0x140010, // Switch 2 (Pokemon Pokopia): 0x530-byte config, UTF-8 text + Version1441810 = 0x160012, // Switch 2 (Fire Emblem: Fortune's Weave): native 0x530-byte config }; enum class SwkbdType : u32 { @@ -184,8 +188,64 @@ struct SwkbdConfigNew { }; static_assert(sizeof(SwkbdConfigNew) == 0x4C8 - sizeof(SwkbdConfigCommon), "SwkbdConfigNew has incorrect size."); + +// SwkbdAppletVersion 0x160012: the text prefix of the common part keeps its layout, the flags, +// lengths, string offsets and dictionary fields move (checked against +// nn::swkbd::keyboardConfig2kbdConfig in Fire Emblem: Fortune's Weave's sdk). +struct SwkbdConfigNative { + std::array<u8, 0x3AA> common_prefix{}; + bool use_utf8{}; + bool use_blur_background{}; + u32 max_text_length{}; + u32 min_text_length{}; + SwkbdPasswordMode password_mode{}; + SwkbdTextDrawType text_draw_type{}; + u32 initial_string_offset{}; + u32 initial_string_length{}; + u32 user_dictionary_offset{}; + u32 user_dictionary_entries{}; + std::array<u64, 24> customized_dictionary_set_entries{}; + u8 total_customized_dictionary_set_entries{}; + bool use_text_check{}; + bool disable_cancel_button{}; + u8 trigger{}; + std::array<u32, 8> text_grouping{}; + INSERT_PADDING_BYTES(0x80); +}; +static_assert(sizeof(SwkbdConfigNative) == 0x530); +static_assert(offsetof(SwkbdConfigNative, use_text_check) == 0x48D); +static_assert(offsetof(SwkbdConfigNative, text_grouping) == 0x490); #pragma pack(pop) +inline bool DecodeNativeSwkbdConfig(std::span<const u8> data, SwkbdConfigCommon& common, + SwkbdConfigNew& config) { + if (data.size() != sizeof(SwkbdConfigNative)) { + return false; + } + SwkbdConfigNative native{}; + std::memcpy(&native, data.data(), sizeof(native)); + common = {}; + config = {}; + std::memcpy(static_cast<void*>(&common), native.common_prefix.data(), native.common_prefix.size()); + common.enable_return_button = native.common_prefix[0x1B] != 0; + common.use_utf8 = native.use_utf8; + common.use_blur_background = native.use_blur_background; + common.max_text_length = native.max_text_length; + common.min_text_length = native.min_text_length; + common.password_mode = native.password_mode; + common.text_draw_type = native.text_draw_type; + common.initial_string_offset = native.initial_string_offset; + common.initial_string_length = native.initial_string_length; + common.user_dictionary_offset = native.user_dictionary_offset; + common.user_dictionary_entries = native.user_dictionary_entries; + common.use_text_check = native.use_text_check; + config.text_grouping = native.text_grouping; + config.customized_dictionary_set_entries = native.customized_dictionary_set_entries; + config.total_customized_dictionary_set_entries = native.total_customized_dictionary_set_entries; + config.disable_cancel_button = native.disable_cancel_button; + return true; +} + struct SwkbdTextCheck { SwkbdTextCheckResult text_check_result{}; std::array<char16_t, STRING_BUFFER_SIZE / 2> text_check_message{}; diff --git a/src/core/hle/service/am/service/application_functions.cpp b/src/core/hle/service/am/service/application_functions.cpp index 37dfbfa344..c63d29b4e9 100644 --- a/src/core/hle/service/am/service/application_functions.cpp +++ b/src/core/hle/service/am/service/application_functions.cpp @@ -58,18 +58,19 @@ IApplicationFunctions::IApplicationFunctions(Core::System& system_, std::shared_ : ServiceFramework{system_, "IApplicationFunctions"}, m_applet{std::move(applet)} { // clang-format off static const FunctionInfo functions[] = { + {0, &IApplicationFunctions::StubReplyZeroed, "Unknown0"}, {1, D<&IApplicationFunctions::PopLaunchParameter>, "PopLaunchParameter"}, - {10, nullptr, "CreateApplicationAndPushAndRequestToStart"}, - {11, nullptr, "CreateApplicationAndPushAndRequestToStartForQuest"}, + {10, &IApplicationFunctions::StubReplyZeroed, "CreateApplicationAndPushAndRequestToStart"}, + {11, &IApplicationFunctions::StubReplyZeroed, "CreateApplicationAndPushAndRequestToStartForQuest"}, {12, D<&IApplicationFunctions::CreateApplicationAndRequestToStart>, "CreateApplicationAndRequestToStart"}, - {13, nullptr, "CreateApplicationAndRequestToStartForQuest"}, - {14, nullptr, "CreateApplicationWithAttributeAndPushAndRequestToStartForQuest"}, - {15, nullptr, "CreateApplicationWithAttributeAndRequestToStartForQuest"}, + {13, &IApplicationFunctions::StubReplyZeroed, "CreateApplicationAndRequestToStartForQuest"}, + {14, &IApplicationFunctions::StubReplyZeroed, "CreateApplicationWithAttributeAndPushAndRequestToStartForQuest"}, + {15, &IApplicationFunctions::StubReplyZeroed, "CreateApplicationWithAttributeAndRequestToStartForQuest"}, {20, D<&IApplicationFunctions::EnsureSaveData>, "EnsureSaveData"}, {21, D<&IApplicationFunctions::GetDesiredLanguage>, "GetDesiredLanguage"}, {22, D<&IApplicationFunctions::SetTerminateResult>, "SetTerminateResult"}, {23, D<&IApplicationFunctions::GetDisplayVersion>, "GetDisplayVersion"}, - {24, nullptr, "GetLaunchStorageInfoForDebug"}, + {24, &IApplicationFunctions::StubReplyZeroed, "GetLaunchStorageInfoForDebug"}, {25, D<&IApplicationFunctions::ExtendSaveData>, "ExtendSaveData"}, {26, D<&IApplicationFunctions::GetSaveDataSize>, "GetSaveDataSize"}, {27, D<&IApplicationFunctions::CreateCacheStorage>, "CreateCacheStorage"}, @@ -80,53 +81,57 @@ IApplicationFunctions::IApplicationFunctions(Core::System& system_, std::shared_ {32, D<&IApplicationFunctions::BeginBlockingHomeButton>, "BeginBlockingHomeButton"}, {33, D<&IApplicationFunctions::EndBlockingHomeButton>, "EndBlockingHomeButton"}, {34, nullptr, "SelectApplicationLicense"}, - {35, nullptr, "GetDeviceSaveDataSizeMax"}, - {36, nullptr, "GetLimitedApplicationLicense"}, - {37, nullptr, "GetLimitedApplicationLicenseUpgradableEvent"}, + {35, &IApplicationFunctions::StubReplyZeroed, "GetDeviceSaveDataSizeMax"}, + {36, &IApplicationFunctions::StubReplyZeroed, "GetLimitedApplicationLicense"}, + {37, &IApplicationFunctions::StubReplyEvent, "GetLimitedApplicationLicenseUpgradableEvent"}, {40, D<&IApplicationFunctions::NotifyRunning>, "NotifyRunning"}, {50, D<&IApplicationFunctions::GetPseudoDeviceId>, "GetPseudoDeviceId"}, {60, D<&IApplicationFunctions::SetMediaPlaybackStateForApplication>, "SetMediaPlaybackStateForApplication"}, {65, D<&IApplicationFunctions::IsGamePlayRecordingSupported>, "IsGamePlayRecordingSupported"}, {66, D<&IApplicationFunctions::InitializeGamePlayRecording>, "InitializeGamePlayRecording"}, {67, D<&IApplicationFunctions::SetGamePlayRecordingState>, "SetGamePlayRecordingState"}, - {68, nullptr, "RequestFlushGamePlayingMovieForDebug"}, - {70, nullptr, "RequestToShutdown"}, - {71, nullptr, "RequestToReboot"}, - {72, nullptr, "RequestToSleep"}, - {80, nullptr, "ExitAndRequestToShowThanksMessage"}, + {68, &IApplicationFunctions::StubReplyZeroed, "RequestFlushGamePlayingMovieForDebug"}, + {70, &IApplicationFunctions::StubReplyZeroed, "RequestToShutdown"}, + {71, &IApplicationFunctions::StubReplyZeroed, "RequestToReboot"}, + {72, &IApplicationFunctions::StubReplyZeroed, "RequestToSleep"}, + {80, &IApplicationFunctions::StubReplyZeroed, "ExitAndRequestToShowThanksMessage"}, {90, D<&IApplicationFunctions::EnableApplicationCrashReport>, "EnableApplicationCrashReport"}, {100, D<&IApplicationFunctions::InitializeApplicationCopyrightFrameBuffer>, "InitializeApplicationCopyrightFrameBuffer"}, {101, D<&IApplicationFunctions::SetApplicationCopyrightImage>, "SetApplicationCopyrightImage"}, {102, D<&IApplicationFunctions::SetApplicationCopyrightVisibility>, "SetApplicationCopyrightVisibility"}, {110, D<&IApplicationFunctions::QueryApplicationPlayStatistics>, "QueryApplicationPlayStatistics"}, {111, D<&IApplicationFunctions::QueryApplicationPlayStatisticsByUid>, "QueryApplicationPlayStatisticsByUid"}, + {112, &IApplicationFunctions::StubReplyZeroed, "Unknown112"}, + {113, &IApplicationFunctions::StubReplyZeroed, "Unknown113"}, {120, D<&IApplicationFunctions::ExecuteProgram>, "ExecuteProgram"}, {121, D<&IApplicationFunctions::ClearUserChannel>, "ClearUserChannel"}, {122, D<&IApplicationFunctions::UnpopToUserChannel>, "UnpopToUserChannel"}, {123, D<&IApplicationFunctions::GetPreviousProgramIndex>, "GetPreviousProgramIndex"}, - {124, nullptr, "EnableApplicationAllThreadDumpOnCrash"}, + {124, &IApplicationFunctions::StubReplyZeroed, "EnableApplicationAllThreadDumpOnCrash"}, {130, D<&IApplicationFunctions::GetGpuErrorDetectedSystemEvent>, "GetGpuErrorDetectedSystemEvent"}, - {131, nullptr, "SetDelayTimeToAbortOnGpuError"}, + {131, &IApplicationFunctions::StubReplyZeroed, "SetDelayTimeToAbortOnGpuError"}, {140, D<&IApplicationFunctions::GetFriendInvitationStorageChannelEvent>, "GetFriendInvitationStorageChannelEvent"}, {141, D<&IApplicationFunctions::TryPopFromFriendInvitationStorageChannel>, "TryPopFromFriendInvitationStorageChannel"}, {150, D<&IApplicationFunctions::GetNotificationStorageChannelEvent>, "GetNotificationStorageChannelEvent"}, - {151, nullptr, "TryPopFromNotificationStorageChannel"}, + {151, &IApplicationFunctions::StubReplyInterface, "TryPopFromNotificationStorageChannel"}, {160, D<&IApplicationFunctions::GetHealthWarningDisappearedSystemEvent>, "GetHealthWarningDisappearedSystemEvent"}, - {170, nullptr, "SetHdcpAuthenticationActivated"}, - {180, nullptr, "GetLaunchRequiredVersion"}, - {181, nullptr, "UpgradeLaunchRequiredVersion"}, - {190, nullptr, "SendServerMaintenanceOverlayNotification"}, - {200, nullptr, "GetLastApplicationExitReason"}, + {170, &IApplicationFunctions::StubReplyZeroed, "SetHdcpAuthenticationActivated"}, + {180, &IApplicationFunctions::StubReplyZeroed, "GetLaunchRequiredVersion"}, + {181, &IApplicationFunctions::StubReplyZeroed, "UpgradeLaunchRequiredVersion"}, + {190, &IApplicationFunctions::StubReplyZeroed, "SendServerMaintenanceOverlayNotification"}, + {200, &IApplicationFunctions::StubReplyZeroed, "GetLastApplicationExitReason"}, {210, D<&IApplicationFunctions::GetOperationModeRefreshEvent>, "GetOperationModeRefreshEvent (guessed)"}, - {220, nullptr, "Unknown220"}, // [20.0.0+] - {300, nullptr, "CreateMovieWriter"}, // [19.0.0+] - {310, nullptr, "Unknown310"}, // [20.0.0+] + {220, &IApplicationFunctions::StubReplyZeroed, "Unknown220"}, // [20.0.0+] + {230, &IApplicationFunctions::StubReplyZeroed, "Unknown230"}, + {300, &IApplicationFunctions::StubReplyInterface, "CreateMovieWriter"}, // [19.0.0+] + {310, &IApplicationFunctions::StubReplyZeroed, "Unknown310"}, // [20.0.0+] {320, D<&IApplicationFunctions::Unknown320>, "Unknown320"}, // [20.0.0+] {321, D<&IApplicationFunctions::Unknown321>, "Unknown321"}, // Switch 2 {330, D<&IApplicationFunctions::Unknown330>, "Unknown330"}, // [20.0.0+] - {500, nullptr, "StartContinuousRecordingFlushForDebug"}, - {1000, nullptr, "CreateMovieMaker"}, + {500, &IApplicationFunctions::StubReplyEvent, "StartContinuousRecordingFlushForDebug"}, + {1000, &IApplicationFunctions::StubReplyInterface, "CreateMovieMaker"}, {1001, D<&IApplicationFunctions::PrepareForJit>, "PrepareForJit"}, + {2000, &IApplicationFunctions::StubReplyZeroed, "Unknown2000"}, }; // clang-format on @@ -571,15 +576,18 @@ Result IApplicationFunctions::GetOperationModeRefreshEvent( R_SUCCEED(); } -Result IApplicationFunctions::Unknown320() { - // Called by Switch 2 titles during nn::oe::Initialize; arguments unknown. +Result IApplicationFunctions::Unknown320(Out<u16> out) { + // Called by Switch 2 titles during nn::oe::Initialize. It returns two bytes of data (meaning + // unknown); a reply without them left the caller reading stale message words. LOG_WARNING(Service_AM, "(STUBBED) called"); + *out = 0; R_SUCCEED(); } -Result IApplicationFunctions::Unknown321() { - // Called by Switch 2 titles right after 320; arguments unknown. +Result IApplicationFunctions::Unknown321(Out<u8> out) { + // Called by Switch 2 titles right after 320; returns one byte (meaning unknown). LOG_WARNING(Service_AM, "(STUBBED) called"); + *out = 0; R_SUCCEED(); } diff --git a/src/core/hle/service/am/service/application_functions.h b/src/core/hle/service/am/service/application_functions.h index 44b5fadff0..aca2f8c694 100644 --- a/src/core/hle/service/am/service/application_functions.h +++ b/src/core/hle/service/am/service/application_functions.h @@ -83,8 +83,8 @@ private: Result GetHealthWarningDisappearedSystemEvent(OutCopyHandle<Kernel::KReadableEvent> out_event); Result CreateApplicationAndRequestToStart(u64 application_id); Result GetOperationModeRefreshEvent(OutCopyHandle<Kernel::KReadableEvent> out_event); - Result Unknown320(); - Result Unknown321(); + Result Unknown320(Out<u16> out); + Result Unknown321(Out<u8> out); Result Unknown330(Out<u8> out); Result PrepareForJit(); diff --git a/src/core/hle/service/am/service/application_proxy_service.cpp b/src/core/hle/service/am/service/application_proxy_service.cpp index 67306aadc0..9d332798b4 100644 --- a/src/core/hle/service/am/service/application_proxy_service.cpp +++ b/src/core/hle/service/am/service/application_proxy_service.cpp @@ -21,6 +21,7 @@ IApplicationProxyService::IApplicationProxyService(Core::System& system_, {0, D<&IApplicationProxyService::OpenApplicationProxy>, "OpenApplicationProxy"}, // Switch 2 SDKs open the proxy with command 10, same arguments. {10, D<&IApplicationProxyService::OpenApplicationProxy>, "OpenApplicationProxy (Switch 2)"}, + {2000, &IApplicationProxyService::StubReplyInterface, "Unknown2000"}, }; RegisterHandlers(functions); } diff --git a/src/core/hle/service/am/service/audio_controller.cpp b/src/core/hle/service/am/service/audio_controller.cpp index 4e9af756f2..7901f67c27 100644 --- a/src/core/hle/service/am/service/audio_controller.cpp +++ b/src/core/hle/service/am/service/audio_controller.cpp @@ -18,7 +18,7 @@ IAudioController::IAudioController(Core::System& system_) {2, D<&IAudioController::GetLibraryAppletExpectedMasterVolume>, "GetLibraryAppletExpectedMasterVolume"}, {3, D<&IAudioController::ChangeMainAppletMasterVolume>, "ChangeMainAppletMasterVolume"}, {4, D<&IAudioController::SetTransparentVolumeRate>, "SetTransparentVolumeRate"}, - {5, nullptr, "Unknown5"}, //20.0.0+ + {5, &IAudioController::StubReplyZeroed, "Unknown5"}, //20.0.0+ }; // clang-format on diff --git a/src/core/hle/service/am/service/common_state_getter.cpp b/src/core/hle/service/am/service/common_state_getter.cpp index 044929e434..2c5e536414 100644 --- a/src/core/hle/service/am/service/common_state_getter.cpp +++ b/src/core/hle/service/am/service/common_state_getter.cpp @@ -24,67 +24,73 @@ ICommonStateGetter::ICommonStateGetter(Core::System& system_, std::shared_ptr<Ap static const FunctionInfo functions[] = { {0, D<&ICommonStateGetter::GetEventHandle>, "GetEventHandle"}, {1, D<&ICommonStateGetter::ReceiveMessage>, "ReceiveMessage"}, - {2, nullptr, "GetThisAppletKind"}, - {3, nullptr, "AllowToEnterSleep"}, - {4, nullptr, "DisallowToEnterSleep"}, + {2, &ICommonStateGetter::StubReplyZeroed, "GetThisAppletKind"}, + {3, &ICommonStateGetter::StubReplyZeroed, "AllowToEnterSleep"}, + {4, &ICommonStateGetter::StubReplyZeroed, "DisallowToEnterSleep"}, {5, D<&ICommonStateGetter::GetOperationMode>, "GetOperationMode"}, {6, D<&ICommonStateGetter::GetPerformanceMode>, "GetPerformanceMode"}, - {7, nullptr, "GetCradleStatus"}, + {7, &ICommonStateGetter::StubReplyZeroed, "GetCradleStatus"}, {8, D<&ICommonStateGetter::GetBootMode>, "GetBootMode"}, {9, D<&ICommonStateGetter::GetCurrentFocusState>, "GetCurrentFocusState"}, {10, D<&ICommonStateGetter::RequestToAcquireSleepLock>, "RequestToAcquireSleepLock"}, {11, D<&ICommonStateGetter::ReleaseSleepLock>, "ReleaseSleepLock"}, {12, D<&ICommonStateGetter::ReleaseSleepLockTransiently>, "ReleaseSleepLockTransiently"}, {13, D<&ICommonStateGetter::GetAcquiredSleepLockEvent>, "GetAcquiredSleepLockEvent"}, - {14, nullptr, "GetWakeupCount"}, //11.0.0+ - {15, nullptr, "Unknown15"}, //19.0.0+ + {14, &ICommonStateGetter::StubReplyZeroed, "GetWakeupCount"}, //11.0.0+ + {15, &ICommonStateGetter::StubReplyEvent, "Unknown15"}, //19.0.0+ {20, D<&ICommonStateGetter::PushToGeneralChannel>, "PushToGeneralChannel"}, {30, D<&ICommonStateGetter::GetHomeButtonReaderLockAccessor>, "GetHomeButtonReaderLockAccessor"}, {31, D<&ICommonStateGetter::GetReaderLockAccessorEx>, "GetReaderLockAccessorEx"}, //2.0.0+ {32, D<&ICommonStateGetter::GetWriterLockAccessorEx>, "GetWriterLockAccessorEx"}, //7.0.0+ - {40, nullptr, "GetCradleFwVersion"}, //2.0.0+ + {40, &ICommonStateGetter::StubReplyZeroed, "GetCradleFwVersion"}, //2.0.0+ {50, D<&ICommonStateGetter::IsVrModeEnabled>, "IsVrModeEnabled"}, //3.0.0+ {51, D<&ICommonStateGetter::SetVrModeEnabled>, "SetVrModeEnabled"}, //3.0.0+ {52, D<&ICommonStateGetter::SetLcdBacklighOffEnabled>, "SetLcdBacklighOffEnabled"}, //4.0.0+ {53, D<&ICommonStateGetter::BeginVrModeEx>, "BeginVrModeEx"}, //7.0.0+ {54, D<&ICommonStateGetter::EndVrModeEx>, "EndVrModeEx"}, //7.0.0+ {55, D<&ICommonStateGetter::IsInControllerFirmwareUpdateSection>, "IsInControllerFirmwareUpdateSection"}, //3.0.0+ - {59, nullptr, "SetVrPositionForDebug"}, //1.0.0+ + {59, &ICommonStateGetter::StubReplyZeroed, "SetVrPositionForDebug"}, //1.0.0+ {60, D<&ICommonStateGetter::GetDefaultDisplayResolution>, "GetDefaultDisplayResolution"}, {61, D<&ICommonStateGetter::GetDefaultDisplayResolutionChangeEvent>, "GetDefaultDisplayResolutionChangeEvent"}, {62, D<&ICommonStateGetter::GetHdcpAuthenticationState>, "GetHdcpAuthenticationState"}, {63, D<&ICommonStateGetter::GetHdcpAuthenticationStateChangeEvent>, "GetHdcpAuthenticationStateChangeEvent"}, - {64, nullptr, "SetTvPowerStateMatchingMode"}, - {65, nullptr, "GetApplicationIdByContentActionName"}, + {64, &ICommonStateGetter::StubReplyZeroed, "SetTvPowerStateMatchingMode"}, + {65, &ICommonStateGetter::StubReplyZeroed, "GetApplicationIdByContentActionName"}, {66, &ICommonStateGetter::SetCpuBoostMode, "SetCpuBoostMode"}, - {67, nullptr, "CancelCpuBoostMode"}, + {67, &ICommonStateGetter::StubReplyZeroed, "CancelCpuBoostMode"}, {68, D<&ICommonStateGetter::GetBuiltInDisplayType>, "GetBuiltInDisplayType"}, {80, D<&ICommonStateGetter::PerformSystemButtonPressingIfInFocus>, "PerformSystemButtonPressingIfInFocus"}, - {90, nullptr, "SetPerformanceConfigurationChangedNotification"}, - {91, nullptr, "GetCurrentPerformanceConfiguration"}, + {90, &ICommonStateGetter::StubReplyZeroed, "SetPerformanceConfigurationChangedNotification"}, + {91, &ICommonStateGetter::StubReplyZeroed, "GetCurrentPerformanceConfiguration"}, {100, D<&ICommonStateGetter::SetHandlingHomeButtonShortPressedEnabled>, "SetHandlingHomeButtonShortPressedEnabled"}, - {110, nullptr, "OpenMyGpuErrorHandler"}, + {110, &ICommonStateGetter::StubReplyInterface, "OpenMyGpuErrorHandler"}, {120, D<&ICommonStateGetter::GetAppletLaunchedHistory>, "GetAppletLaunchedHistory"}, //13.0.0+ {130, D<&ICommonStateGetter::EnableStartupLogoDisappearedMessage>, "EnableStartupLogoDisappearedMessage"}, //21.0.0+ {200, D<&ICommonStateGetter::GetOperationModeSystemInfo>, "GetOperationModeSystemInfo"}, {300, D<&ICommonStateGetter::GetSettingsPlatformRegion>, "GetSettingsPlatformRegion"}, - {400, nullptr, "ActivateMigrationService"}, - {401, nullptr, "DeactivateMigrationService"}, - {500, nullptr, "DisableSleepTillShutdown"}, - {501, nullptr, "SuppressDisablingSleepTemporarily"}, - {502, nullptr, "IsSleepEnabled"}, - {503, nullptr, "IsDisablingSleepSuppressed"}, - {600, nullptr, "SetHidInputMagnificationForApplication"}, //20.0.0+ + {310, &ICommonStateGetter::StubReplyZeroed, "Unknown310"}, + {320, &ICommonStateGetter::StubReplyZeroed, "Unknown320"}, + {400, &ICommonStateGetter::StubReplyZeroed, "ActivateMigrationService"}, + {401, &ICommonStateGetter::StubReplyZeroed, "DeactivateMigrationService"}, + {500, &ICommonStateGetter::StubReplyZeroed, "DisableSleepTillShutdown"}, + {501, &ICommonStateGetter::StubReplyZeroed, "SuppressDisablingSleepTemporarily"}, + {502, &ICommonStateGetter::StubReplyZeroed, "IsSleepEnabled"}, + {503, &ICommonStateGetter::StubReplyZeroed, "IsDisablingSleepSuppressed"}, + {600, &ICommonStateGetter::StubReplyZeroed, "SetHidInputMagnificationForApplication"}, //20.0.0+ {610, D<&ICommonStateGetter::Unknown610>, "Unknown610"}, //21.0.0+ {611, D<&ICommonStateGetter::Unknown611>, "Unknown611"}, //22.0.0+ + {650, &ICommonStateGetter::StubReplyZeroed, "Unknown650"}, + {651, &ICommonStateGetter::StubReplyZeroed, "Unknown651"}, {900, D<&ICommonStateGetter::SetRequestExitToLibraryAppletAtExecuteNextProgramEnabled>, "SetRequestExitToLibraryAppletAtExecuteNextProgramEnabled"}, //11.0.0+ - {910, nullptr, "GetLaunchRequiredTick"}, //17.0.0+ + {910, &ICommonStateGetter::StubReplyZeroed, "GetLaunchRequiredTick"}, //17.0.0+ {1000, D<&ICommonStateGetter::BeginVrMode3d>, "BeginVrMode3d"}, //19.0.0+ {1001, D<&ICommonStateGetter::EndVrMode3d>, "EndVrMode3d"}, //19.0.0+ {1002, D<&ICommonStateGetter::IsVrModeEnabled3d>, "IsVrModeEnabled3d"}, //19.0.0+ {1003, D<&ICommonStateGetter::GetVrLaboGoggleViewport>, "GetVrLaboGoggleViewport"}, //21.0.0+ {1004, D<&ICommonStateGetter::GetPanelPhysicalSizeForSpecificTitle>, "GetPanelPhysicalSizeForSpecificTitle"}, //21.0.0+ {1005, D<&ICommonStateGetter::GetPanelResolutionForSpecificTitle>, "GetPanelResolutionForSpecificTitle"}, //21.0.0+ + {2000, &ICommonStateGetter::StubReplyZeroed, "Unknown2000"}, + {2010, &ICommonStateGetter::StubReplyZeroed, "Unknown2010"}, }; // clang-format on diff --git a/src/core/hle/service/am/service/display_controller.cpp b/src/core/hle/service/am/service/display_controller.cpp index 8f87327b17..7e7db23708 100644 --- a/src/core/hle/service/am/service/display_controller.cpp +++ b/src/core/hle/service/am/service/display_controller.cpp @@ -16,34 +16,35 @@ IDisplayController::IDisplayController(Core::System& system_, std::shared_ptr<Ap : ServiceFramework{system_, "IDisplayController"}, applet(std::move(applet_)) { // clang-format off static const FunctionInfo functions[] = { - {0, nullptr, "GetLastForegroundCaptureImage"}, - {1, nullptr, "UpdateLastForegroundCaptureImage"}, - {2, nullptr, "GetLastApplicationCaptureImage"}, - {3, nullptr, "GetCallerAppletCaptureImage"}, - {4, nullptr, "UpdateCallerAppletCaptureImage"}, + {0, &IDisplayController::StubReplyZeroed, "GetLastForegroundCaptureImage"}, + {1, &IDisplayController::StubReplyZeroed, "UpdateLastForegroundCaptureImage"}, + {2, &IDisplayController::StubReplyZeroed, "GetLastApplicationCaptureImage"}, + {3, &IDisplayController::StubReplyZeroed, "GetCallerAppletCaptureImage"}, + {4, &IDisplayController::StubReplyZeroed, "UpdateCallerAppletCaptureImage"}, {5, D<&IDisplayController::GetLastForegroundCaptureImageEx>, "GetLastForegroundCaptureImageEx"}, - {6, nullptr, "GetLastApplicationCaptureImageEx"}, + {6, &IDisplayController::StubReplyZeroed, "GetLastApplicationCaptureImageEx"}, {7, D<&IDisplayController::GetCallerAppletCaptureImageEx>, "GetCallerAppletCaptureImageEx"}, {8, D<&IDisplayController::TakeScreenShotOfOwnLayer>, "TakeScreenShotOfOwnLayer"}, - {9, nullptr, "CopyBetweenCaptureBuffers"}, - {10, nullptr, "AcquireLastApplicationCaptureBuffer"}, - {11, nullptr, "ReleaseLastApplicationCaptureBuffer"}, - {12, nullptr, "AcquireLastForegroundCaptureBuffer"}, - {13, nullptr, "ReleaseLastForegroundCaptureBuffer"}, - {14, nullptr, "AcquireCallerAppletCaptureBuffer"}, - {15, nullptr, "ReleaseCallerAppletCaptureBuffer"}, - {16, nullptr, "AcquireLastApplicationCaptureBufferEx"}, - {17, nullptr, "AcquireLastForegroundCaptureBufferEx"}, - {18, nullptr, "AcquireCallerAppletCaptureBufferEx"}, + {9, &IDisplayController::StubReplyZeroed, "CopyBetweenCaptureBuffers"}, + {10, &IDisplayController::StubReplyEvent, "AcquireLastApplicationCaptureBuffer"}, + {11, &IDisplayController::StubReplyZeroed, "ReleaseLastApplicationCaptureBuffer"}, + {12, &IDisplayController::StubReplyEvent, "AcquireLastForegroundCaptureBuffer"}, + {13, &IDisplayController::StubReplyZeroed, "ReleaseLastForegroundCaptureBuffer"}, + {14, &IDisplayController::StubReplyEvent, "AcquireCallerAppletCaptureBuffer"}, + {15, &IDisplayController::StubReplyZeroed, "ReleaseCallerAppletCaptureBuffer"}, + {16, &IDisplayController::StubReplyEvent, "AcquireLastApplicationCaptureBufferEx"}, + {17, &IDisplayController::StubReplyEvent, "AcquireLastForegroundCaptureBufferEx"}, + {18, &IDisplayController::StubReplyEvent, "AcquireCallerAppletCaptureBufferEx"}, + {19, &IDisplayController::StubReplyZeroed, "Unknown19"}, {20, D<&IDisplayController::ClearCaptureBuffer>, "ClearCaptureBuffer"}, - {21, nullptr, "ClearAppletTransitionBuffer"}, + {21, &IDisplayController::StubReplyZeroed, "ClearAppletTransitionBuffer"}, {22, D<&IDisplayController::AcquireLastApplicationCaptureSharedBuffer>, "AcquireLastApplicationCaptureSharedBuffer"}, {23, D<&IDisplayController::ReleaseLastApplicationCaptureSharedBuffer>, "ReleaseLastApplicationCaptureSharedBuffer"}, {24, D<&IDisplayController::AcquireLastForegroundCaptureSharedBuffer>, "AcquireLastForegroundCaptureSharedBuffer"}, {25, D<&IDisplayController::ReleaseLastForegroundCaptureSharedBuffer>, "ReleaseLastForegroundCaptureSharedBuffer"}, {26, D<&IDisplayController::AcquireCallerAppletCaptureSharedBuffer>, "AcquireCallerAppletCaptureSharedBuffer"}, {27, D<&IDisplayController::ReleaseCallerAppletCaptureSharedBuffer>, "ReleaseCallerAppletCaptureSharedBuffer"}, - {28, nullptr, "TakeScreenShotOfOwnLayerEx"}, + {28, &IDisplayController::StubReplyZeroed, "TakeScreenShotOfOwnLayerEx"}, }; // clang-format on diff --git a/src/core/hle/service/am/service/graphics_scale_controller.h b/src/core/hle/service/am/service/graphics_scale_controller.h index 07e38c6ac8..b31858d3ed 100644 --- a/src/core/hle/service/am/service/graphics_scale_controller.h +++ b/src/core/hle/service/am/service/graphics_scale_controller.h @@ -25,9 +25,13 @@ public: } private: - Result GetCurrentGraphicsScale(Out<u32> out_scale) { + // The reply is 12 bytes; only the first word (the scale) is known. The other two used to be + // left to the message buffer's leftovers. + Result GetCurrentGraphicsScale(Out<u32> out_scale, Out<u32> out_unknown1, Out<u32> out_unknown2) { LOG_WARNING(Service_AM, "(STUBBED) called, returning 10"); *out_scale = 10; + *out_unknown1 = 0; + *out_unknown2 = 0; R_SUCCEED(); } }; diff --git a/src/core/hle/service/am/service/library_applet_accessor.cpp b/src/core/hle/service/am/service/library_applet_accessor.cpp index 849e5794cf..7119c60596 100644 --- a/src/core/hle/service/am/service/library_applet_accessor.cpp +++ b/src/core/hle/service/am/service/library_applet_accessor.cpp @@ -62,21 +62,22 @@ ILibraryAppletAccessor::ILibraryAppletAccessor(Core::System& system_, {20, D<&ILibraryAppletAccessor::RequestExit>, "RequestExit"}, {25, D<&ILibraryAppletAccessor::Terminate>, "Terminate"}, {30, D<&ILibraryAppletAccessor::GetResult>, "GetResult"}, - {50, nullptr, "SetOutOfFocusApplicationSuspendingEnabled"}, + {50, &ILibraryAppletAccessor::StubReplyZeroed, "SetOutOfFocusApplicationSuspendingEnabled"}, {60, D<&ILibraryAppletAccessor::PresetLibraryAppletGpuTimeSliceZero>, "PresetLibraryAppletGpuTimeSliceZero"}, //10.0.0+ - {80, nullptr, "RequestForLibraryAppletToGetForeground"}, //19.0.0+ - {81, nullptr, "GetCurrentChildLibraryApplet"}, //19.0.0+ + {70, &ILibraryAppletAccessor::StubReplyZeroed, "Unknown70"}, + {80, &ILibraryAppletAccessor::StubReplyZeroed, "RequestForLibraryAppletToGetForeground"}, //19.0.0+ + {81, &ILibraryAppletAccessor::StubReplyInterface, "GetCurrentChildLibraryApplet"}, //19.0.0+ {90, D<&ILibraryAppletAccessor::Unknown90>, "Unknown90"}, //20.0.0+ {100, D<&ILibraryAppletAccessor::PushInData>, "PushInData"}, {101, D<&ILibraryAppletAccessor::PopOutData>, "PopOutData"}, - {102, nullptr, "PushExtraStorage"}, + {102, &ILibraryAppletAccessor::StubReplyZeroed, "PushExtraStorage"}, {103, D<&ILibraryAppletAccessor::PushInteractiveInData>, "PushInteractiveInData"}, {104, D<&ILibraryAppletAccessor::PopInteractiveOutData>, "PopInteractiveOutData"}, {105, D<&ILibraryAppletAccessor::GetPopOutDataEvent>, "GetPopOutDataEvent"}, {106, D<&ILibraryAppletAccessor::GetPopInteractiveOutDataEvent>, "GetPopInteractiveOutDataEvent"}, - {110, nullptr, "NeedsToExitProcess"}, + {110, &ILibraryAppletAccessor::StubReplyZeroed, "NeedsToExitProcess"}, {120, D<&ILibraryAppletAccessor::GetLibraryAppletInfo>, "GetLibraryAppletInfo"}, - {150, nullptr, "RequestForAppletToGetForeground"}, + {150, &ILibraryAppletAccessor::StubReplyZeroed, "RequestForAppletToGetForeground"}, {160, D<&ILibraryAppletAccessor::GetIndirectLayerConsumerHandle>, "GetIndirectLayerConsumerHandle"}, //2.0.0+ {170, D<&ILibraryAppletAccessor::Unknown170>, "Unknown170"}, //22.0.0+ }; diff --git a/src/core/hle/service/am/service/library_applet_creator.cpp b/src/core/hle/service/am/service/library_applet_creator.cpp index afdb609c00..2993ed7436 100644 --- a/src/core/hle/service/am/service/library_applet_creator.cpp +++ b/src/core/hle/service/am/service/library_applet_creator.cpp @@ -204,8 +204,8 @@ ILibraryAppletCreator::ILibraryAppletCreator(Core::System& system_, std::shared_ m_window_system{window_system}, m_applet{std::move(applet)} { static const FunctionInfo functions[] = { {0, D<&ILibraryAppletCreator::CreateLibraryApplet>, "CreateLibraryApplet"}, - {1, nullptr, "TerminateAllLibraryApplets"}, - {2, nullptr, "AreAnyLibraryAppletsLeft"}, + {1, &ILibraryAppletCreator::StubReplyZeroed, "TerminateAllLibraryApplets"}, + {2, &ILibraryAppletCreator::StubReplyZeroed, "AreAnyLibraryAppletsLeft"}, {3, D<&ILibraryAppletCreator::CreateLibraryAppletEx>, "CreateLibraryAppletEx"}, {10, D<&ILibraryAppletCreator::CreateStorage>, "CreateStorage"}, {11, D<&ILibraryAppletCreator::CreateTransferMemoryStorage>, "CreateTransferMemoryStorage"}, diff --git a/src/core/hle/service/am/service/self_controller.cpp b/src/core/hle/service/am/service/self_controller.cpp index a8078e6048..d67e1b32cd 100644 --- a/src/core/hle/service/am/service/self_controller.cpp +++ b/src/core/hle/service/am/service/self_controller.cpp @@ -37,44 +37,59 @@ ISelfController::ISelfController(Core::System& system_, std::shared_ptr<Applet> {14, D<&ISelfController::SetRestartMessageEnabled>, "SetRestartMessageEnabled"}, {15, D<&ISelfController::SetScreenShotAppletIdentityInfo>, "SetScreenShotAppletIdentityInfo"}, {16, D<&ISelfController::SetOutOfFocusSuspendingEnabled>, "SetOutOfFocusSuspendingEnabled"}, - {17, nullptr, "SetControllerFirmwareUpdateSection"}, - {18, nullptr, "SetRequiresCaptureButtonShortPressedMessage"}, + {17, &ISelfController::StubReplyZeroed, "SetControllerFirmwareUpdateSection"}, + {18, &ISelfController::StubReplyZeroed, "SetRequiresCaptureButtonShortPressedMessage"}, {19, D<&ISelfController::SetAlbumImageOrientation>, "SetAlbumImageOrientation"}, - {20, nullptr, "SetDesirableKeyboardLayout"}, - {21, nullptr, "GetScreenShotProgramId"}, + {20, &ISelfController::StubReplyZeroed, "SetDesirableKeyboardLayout"}, + {21, &ISelfController::StubReplyZeroed, "GetScreenShotProgramId"}, + {22, &ISelfController::StubReplyZeroed, "Unknown22"}, + {23, &ISelfController::StubReplyZeroed, "Unknown23"}, + {24, &ISelfController::StubReplyZeroed, "Unknown24"}, {40, D<&ISelfController::CreateManagedDisplayLayer>, "CreateManagedDisplayLayer"}, {41, D<&ISelfController::IsSystemBufferSharingEnabled>, "IsSystemBufferSharingEnabled"}, {42, D<&ISelfController::GetSystemSharedLayerHandle>, "GetSystemSharedLayerHandle"}, {43, D<&ISelfController::GetSystemSharedBufferHandle>, "GetSystemSharedBufferHandle"}, {44, D<&ISelfController::CreateManagedDisplaySeparableLayer>, "CreateManagedDisplaySeparableLayer"}, - {45, nullptr, "SetManagedDisplayLayerSeparationMode"}, - {46, nullptr, "SetRecordingLayerCompositionEnabled"}, + {45, &ISelfController::StubReplyZeroed, "SetManagedDisplayLayerSeparationMode"}, + {46, &ISelfController::StubReplyZeroed, "SetRecordingLayerCompositionEnabled"}, {50, D<&ISelfController::SetHandlesRequestToDisplay>, "SetHandlesRequestToDisplay"}, {51, D<&ISelfController::ApproveToDisplay>, "ApproveToDisplay"}, {60, D<&ISelfController::OverrideAutoSleepTimeAndDimmingTime>, "OverrideAutoSleepTimeAndDimmingTime"}, {61, D<&ISelfController::SetMediaPlaybackState>, "SetMediaPlaybackState"}, {62, D<&ISelfController::SetIdleTimeDetectionExtension>, "SetIdleTimeDetectionExtension"}, {63, D<&ISelfController::GetIdleTimeDetectionExtension>, "GetIdleTimeDetectionExtension"}, - {64, nullptr, "SetInputDetectionSourceSet"}, + {64, &ISelfController::StubReplyZeroed, "SetInputDetectionSourceSet"}, {65, D<&ISelfController::ReportUserIsActive>, "ReportUserIsActive"}, - {66, nullptr, "GetCurrentIlluminance"}, + {66, &ISelfController::StubReplyZeroed, "GetCurrentIlluminance"}, {67, D<&ISelfController::IsIlluminanceAvailable>, "IsIlluminanceAvailable"}, {68, D<&ISelfController::SetAutoSleepDisabled>, "SetAutoSleepDisabled"}, {69, D<&ISelfController::IsAutoSleepDisabled>, "IsAutoSleepDisabled"}, - {70, nullptr, "ReportMultimediaError"}, - {71, nullptr, "GetCurrentIlluminanceEx"}, + {70, &ISelfController::StubReplyZeroed, "ReportMultimediaError"}, + {71, &ISelfController::StubReplyZeroed, "GetCurrentIlluminanceEx"}, {72, D<&ISelfController::SetInputDetectionPolicy>, "SetInputDetectionPolicy"}, - {80, nullptr, "SetWirelessPriorityMode"}, + {73, &ISelfController::StubReplyZeroed, "Unknown73"}, + {80, &ISelfController::StubReplyZeroed, "SetWirelessPriorityMode"}, {90, D<&ISelfController::GetAccumulatedSuspendedTickValue>, "GetAccumulatedSuspendedTickValue"}, {91, D<&ISelfController::GetAccumulatedSuspendedTickChangedEvent>, "GetAccumulatedSuspendedTickChangedEvent"}, {100, D<&ISelfController::SetAlbumImageTakenNotificationEnabled>, "SetAlbumImageTakenNotificationEnabled"}, - {110, nullptr, "SetApplicationAlbumUserData"}, + {110, &ISelfController::StubReplyZeroed, "SetApplicationAlbumUserData"}, {120, D<&ISelfController::SaveCurrentScreenshot>, "SaveCurrentScreenshot"}, {130, D<&ISelfController::SetRecordVolumeMuted>, "SetRecordVolumeMuted"}, + {200, &ISelfController::StubReplyZeroed, "Unknown200"}, + {210, &ISelfController::StubReplyZeroed, "Unknown210"}, + {211, &ISelfController::StubReplyZeroed, "Unknown211"}, + {220, &ISelfController::StubReplyZeroed, "Unknown220"}, + {221, &ISelfController::StubReplyZeroed, "Unknown221"}, {230, D<&ISelfController::Unknown230>, "Unknown230"}, {240, D<&ISelfController::Unknown240>, "Unknown240"}, - {1000, nullptr, "GetDebugStorageChannel"}, + {1000, &ISelfController::StubReplyInterface, "GetDebugStorageChannel"}, {2000, &ISelfController::GetNx2GraphicsSystemSession, "GetGraphicsSystemSession (Switch 2, guessed)"}, + {2100, &ISelfController::StubReplyZeroed, "Unknown2100"}, + {2200, &ISelfController::StubReplyZeroed, "Unknown2200"}, + {2300, &ISelfController::StubReplyZeroed, "Unknown2300"}, + {2400, &ISelfController::StubReplyZeroed, "Unknown2400"}, + {2401, &ISelfController::StubReplyZeroed, "Unknown2401"}, + {2402, &ISelfController::StubReplyZeroed, "Unknown2402"}, }; // clang-format on diff --git a/src/core/hle/service/am/service/window_controller.cpp b/src/core/hle/service/am/service/window_controller.cpp index f2b9dccb27..20bf71083e 100644 --- a/src/core/hle/service/am/service/window_controller.cpp +++ b/src/core/hle/service/am/service/window_controller.cpp @@ -18,7 +18,7 @@ IWindowController::IWindowController(Core::System& system_, std::shared_ptr<Appl m_window_system{window_system}, m_applet{std::move(applet)} { // clang-format off static const FunctionInfo functions[] = { - {0, nullptr, "CreateWindow"}, + {0, &IWindowController::StubReplyInterface, "CreateWindow"}, {1, D<&IWindowController::GetAppletResourceUserId>, "GetAppletResourceUserId"}, {2, D<&IWindowController::GetAppletResourceUserIdOfCallerApplet>, "GetAppletResourceUserIdOfCallerApplet"}, {10, D<&IWindowController::AcquireForegroundRights>, "AcquireForegroundRights"}, diff --git a/src/core/hle/service/aoc/addon_content_manager.cpp b/src/core/hle/service/aoc/addon_content_manager.cpp index be2aa34baa..c63b08e40d 100644 --- a/src/core/hle/service/aoc/addon_content_manager.cpp +++ b/src/core/hle/service/aoc/addon_content_manager.cpp @@ -68,19 +68,19 @@ IAddOnContentManager::IAddOnContentManager(Core::System& system_) {6, nullptr, "PrepareAddOnContentByApplicationId"}, {7, D<&IAddOnContentManager::PrepareAddOnContent>, "PrepareAddOnContent"}, {8, D<&IAddOnContentManager::GetAddOnContentListChangedEvent>, "GetAddOnContentListChangedEvent"}, - {9, nullptr, "GetAddOnContentLostErrorCode"}, + {9, &IAddOnContentManager::StubReplyZeroed, "GetAddOnContentLostErrorCode"}, {10, D<&IAddOnContentManager::GetAddOnContentListChangedEventWithProcessId>, "GetAddOnContentListChangedEventWithProcessId"}, {11, D<&IAddOnContentManager::NotifyMountAddOnContent>, "NotifyMountAddOnContent"}, {12, D<&IAddOnContentManager::NotifyUnmountAddOnContent>, "NotifyUnmountAddOnContent"}, - {13, nullptr, "IsAddOnContentMountedForDebug"}, + {13, &IAddOnContentManager::StubReplyZeroed, "IsAddOnContentMountedForDebug"}, {50, D<&IAddOnContentManager::CheckAddOnContentMountStatus>, "CheckAddOnContentMountStatus"}, {100, D<&IAddOnContentManager::CreateEcPurchasedEventManager>, "CreateEcPurchasedEventManager"}, {101, D<&IAddOnContentManager::CreatePermanentEcPurchasedEventManager>, "CreatePermanentEcPurchasedEventManager"}, - {110, nullptr, "CreateContentsServiceManager"}, - {200, nullptr, "SetRequiredAddOnContentsOnContentsAvailabilityTransition"}, - {300, nullptr, "SetupHostAddOnContent"}, - {301, nullptr, "GetRegisteredAddOnContentPath"}, - {302, nullptr, "UpdateCachedList"}, + {110, &IAddOnContentManager::StubReplyInterface, "CreateContentsServiceManager"}, + {200, &IAddOnContentManager::StubReplyZeroed, "SetRequiredAddOnContentsOnContentsAvailabilityTransition"}, + {300, &IAddOnContentManager::StubReplyZeroed, "SetupHostAddOnContent"}, + {301, &IAddOnContentManager::StubReplyZeroed, "GetRegisteredAddOnContentPath"}, + {302, &IAddOnContentManager::StubReplyZeroed, "UpdateCachedList"}, }; // clang-format on diff --git a/src/core/hle/service/audio/audio.cpp b/src/core/hle/service/audio/audio.cpp index 211e1cb6bc..eb1511857b 100644 --- a/src/core/hle/service/audio/audio.cpp +++ b/src/core/hle/service/audio/audio.cpp @@ -6,6 +6,7 @@ #include "core/core.h" #include "core/hle/service/audio/audio.h" +#include "core/hle/service/audio/nx2_audio_device_manager.h" #include "core/hle/service/audio/nx2_audio_user_client.h" #include "core/hle/service/audio/audio_controller.h" #include "core/hle/service/audio/audio_in_manager.h" @@ -206,6 +207,8 @@ void LoopProcess(Core::System& system) { server_manager->RegisterNamedService("audren:u", std::make_shared<IAudioRendererManager>(system), 30); // Switch 2 audio entry point (nn::audio::CreateUserClientSessionByHipc). server_manager->RegisterNamedService("aud:u", std::make_shared<INx2AudioUserClient>(system), 30); + server_manager->RegisterNamedService("auddmg:u", std::make_shared<INx2AudioDeviceManagerService>(system), 30); + server_manager->RegisterNamedService("audsmx:u", std::make_shared<INx2AudioSoundMixerService>(system), 30); server_manager->RegisterNamedService("audren:a", std::make_shared<IAudioRendererManagerForApplet>(system), 30); server_manager->RegisterNamedService("hwopus", std::make_shared<IHardwareOpusDecoderManager>(system), 25); ServerManager::RunServer(std::move(server_manager)); diff --git a/src/core/hle/service/audio/audio_device.cpp b/src/core/hle/service/audio/audio_device.cpp index e9f7e074a1..8f5856b76c 100644 --- a/src/core/hle/service/audio/audio_device.cpp +++ b/src/core/hle/service/audio/audio_device.cpp @@ -16,7 +16,9 @@ IAudioDevice::IAudioDevice(Core::System& system_, u64 applet_resource_user_id, u u32 device_num) : ServiceFramework{system_, "IAudioDevice"}, service_context{system_, "IAudioDevice"}, impl{std::make_unique<AudioDevice>(system_, applet_resource_user_id, revision)}, - event{service_context.CreateEvent(fmt::format("IAudioDeviceEvent-{}", device_num))} { + event{service_context.CreateEvent(fmt::format("IAudioDeviceEvent-{}", device_num))}, + notification_event{service_context.CreateEvent( + fmt::format("IAudioDeviceNotification-{}", device_num))} { static const FunctionInfo functions[] = { {0, D<&IAudioDevice::ListAudioDeviceName>, "ListAudioDeviceName"}, {1, D<&IAudioDevice::SetAudioDeviceOutputVolume>, "SetAudioDeviceOutputVolume"}, @@ -27,18 +29,19 @@ IAudioDevice::IAudioDevice(Core::System& system_, u64 applet_resource_user_id, u {6, D<&IAudioDevice::ListAudioDeviceNameAuto>, "ListAudioDeviceNameAuto"}, {7, D<&IAudioDevice::SetAudioDeviceOutputVolumeAuto>, "SetAudioDeviceOutputVolumeAuto"}, {8, D<&IAudioDevice::GetAudioDeviceOutputVolumeAuto>, "GetAudioDeviceOutputVolumeAuto"}, + {9, &IAudioDevice::StubReplyZeroed, "Unknown9"}, {10, D<&IAudioDevice::GetActiveAudioDeviceNameAuto>, "GetActiveAudioDeviceNameAuto"}, {11, D<&IAudioDevice::QueryAudioDeviceInputEvent>, "QueryAudioDeviceInputEvent"}, {12, D<&IAudioDevice::QueryAudioDeviceOutputEvent>, "QueryAudioDeviceOutputEvent"}, {13, D<&IAudioDevice::GetActiveAudioDeviceName>, "GetActiveAudioOutputDeviceName"}, {14, D<&IAudioDevice::ListAudioOutputDeviceName>, "ListAudioOutputDeviceName"}, - {15, nullptr, "AcquireAudioInputDeviceNotification"}, //17.0.0+ - {16, nullptr, "ReleaseAudioInputDeviceNotification"}, //17.0.0+ - {17, nullptr, "AcquireAudioOutputDeviceNotification"}, //17.0.0+ - {18, nullptr, "ReleaseAudioOutputDeviceNotification"}, //17.0.0+ + {15, D<&IAudioDevice::AcquireAudioDeviceNotification>, "AcquireAudioInputDeviceNotification"}, //17.0.0+ + {16, D<&IAudioDevice::ReleaseAudioDeviceNotification>, "ReleaseAudioInputDeviceNotification"}, //17.0.0+ + {17, D<&IAudioDevice::AcquireAudioDeviceNotification>, "AcquireAudioOutputDeviceNotification"}, //17.0.0+ + {18, D<&IAudioDevice::ReleaseAudioDeviceNotification>, "ReleaseAudioOutputDeviceNotification"}, //17.0.0+ {19, D<&IAudioDevice::SetAudioDeviceOutputVolumeAutoTuneEnabled>, "SetAudioDeviceOutputVolumeAutoTuneEnabled"}, //18.0.0+ {20, D<&IAudioDevice::IsAudioDeviceOutputVolumeAutoTuneEnabled>, "IsAudioDeviceOutputVolumeAutoTuneEnabled"}, //18.0.0+ - {21, nullptr, "IsActiveOutputDeviceEstimatedLowLatency"} //21.0.0+ + {21, &IAudioDevice::StubReplyZeroed, "IsActiveOutputDeviceEstimatedLowLatency"} //21.0.0+ }; RegisterHandlers(functions); @@ -46,6 +49,7 @@ IAudioDevice::IAudioDevice(Core::System& system_, u64 applet_resource_user_id, u } IAudioDevice::~IAudioDevice() { + service_context.CloseEvent(notification_event); service_context.CloseEvent(event); } @@ -145,6 +149,21 @@ Result IAudioDevice::QueryAudioDeviceOutputEvent(OutCopyHandle<Kernel::KReadable R_SUCCEED(); } +// 15/17(u64): an event that fires when the input/output device set changes, which it never +// does here. Switch 2 titles (Kirby Air Riders, Switch Sports) ask for it; the auto stub +// replied without a handle. +Result IAudioDevice::AcquireAudioDeviceNotification(OutCopyHandle<Kernel::KReadableEvent> out_event, + u64 unknown) { + LOG_DEBUG(Service_Audio, "called, unknown={:#x}", unknown); + *out_event = ¬ification_event->GetReadableEvent(); + R_SUCCEED(); +} + +Result IAudioDevice::ReleaseAudioDeviceNotification(u64 unknown) { + LOG_DEBUG(Service_Audio, "called, unknown={:#x}", unknown); + R_SUCCEED(); +} + Result IAudioDevice::GetActiveChannelCount(Out<u32> out_active_channel_count) { *out_active_channel_count = system.AudioCore().GetOutputSink().GetSystemChannels(); LOG_DEBUG(Service_Audio, "(STUBBED) called. Channels={}", *out_active_channel_count); diff --git a/src/core/hle/service/audio/audio_device.h b/src/core/hle/service/audio/audio_device.h index f6c12fbfeb..8a7b9a70b0 100644 --- a/src/core/hle/service/audio/audio_device.h +++ b/src/core/hle/service/audio/audio_device.h @@ -54,10 +54,14 @@ private: Out<s32> out_count); Result SetAudioDeviceOutputVolumeAutoTuneEnabled(bool enabled); Result IsAudioDeviceOutputVolumeAutoTuneEnabled(Out<bool> out_enabled); + Result AcquireAudioDeviceNotification(OutCopyHandle<Kernel::KReadableEvent> out_event, + u64 unknown); + Result ReleaseAudioDeviceNotification(u64 unknown); KernelHelpers::ServiceContext service_context; std::unique_ptr<AudioCore::Renderer::AudioDevice> impl; Kernel::KEvent* event; + Kernel::KEvent* notification_event; }; } // namespace Service::Audio diff --git a/src/core/hle/service/audio/audio_renderer_manager.cpp b/src/core/hle/service/audio/audio_renderer_manager.cpp index 97d9f44ddd..d669d64cec 100644 --- a/src/core/hle/service/audio/audio_renderer_manager.cpp +++ b/src/core/hle/service/audio/audio_renderer_manager.cpp @@ -27,7 +27,7 @@ IAudioRendererManager::IAudioRendererManager(Core::System& system_) {0, D<&IAudioRendererManager::OpenAudioRenderer>, "OpenAudioRenderer"}, {1, D<&IAudioRendererManager::GetWorkBufferSize>, "GetWorkBufferSize"}, {2, D<&IAudioRendererManager::GetAudioDeviceService>, "GetAudioDeviceService"}, - {3, nullptr, "OpenAudioRendererForManualExecution"}, + {3, &IAudioRendererManager::StubReplyInterface, "OpenAudioRendererForManualExecution"}, {4, D<&IAudioRendererManager::GetAudioDeviceServiceWithRevisionInfo>, "GetAudioDeviceServiceWithRevisionInfo"}, }; // clang-format on diff --git a/src/core/hle/service/audio/nx2_audio_device_manager.h b/src/core/hle/service/audio/nx2_audio_device_manager.h new file mode 100644 index 0000000000..746454660e --- /dev/null +++ b/src/core/hle/service/audio/nx2_audio_device_manager.h @@ -0,0 +1,539 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <algorithm> +#include <array> +#include <atomic> +#include <chrono> +#include <cstring> +#include <map> +#include <memory> +#include <mutex> +#include <optional> +#include <string_view> +#include <vector> + +#include <fmt/ranges.h> + +#include "common/string_util.h" +#include "core/core.h" +#include "core/core_timing.h" +#include "core/hle/kernel/k_event.h" +#include "core/memory.h" +#include "core/hle/service/ipc_helpers.h" +#include "core/hle/service/kernel_helpers.h" +#include "core/hle/service/service.h" + +namespace Service::Audio { + +// Switch 2 sound mixer streams (nn::audio::sysmix, behind audsmx:u). A stream is a ring buffer in +// the game's own memory: CreateStream hands over its address and size (4 KiB aligned), OpenStream +// its format and length in frames, and once the streams are engaged the system takes frames out of +// the ring at the stream's sample rate. Eden doesn't mix these into its output yet; it drains the +// rings in real time so the game sees its audio being played (Switch Sports' menu movie feeds one). +// The ring's shared indices, in frames modulo (frame count + 1): the system's read index at +// buffer + 0x44, the game's write index at buffer + 0x1004. +struct Nx2SysmixStream { + u32 id{}; + u32 owner{}; ///< the user session whose timer drains it + u64 buffer{}; + u64 size{}; + u32 sample_type{}; + u32 channels{}; + u32 sample_rate{}; + u32 frames{}; + bool open{}; + bool engaged{}; + double owed_frames{}; + u32 data_mode{}; ///< command 7's mode; 0 leaves the data event alone + s32 data_threshold{}; ///< free frames that signal the data event + Kernel::KEvent* state_event{}; + Kernel::KEvent* data_event{}; +}; + +struct Nx2SysmixStreams { + std::mutex mutex; + std::map<u32, std::shared_ptr<Nx2SysmixStream>> streams; + u32 next_id{1}; +}; + +// Streams are created through audsmx:u but started and stopped through auddmg:u (AttachStream and +// DetachStream), so both services share one list. +inline std::shared_ptr<Nx2SysmixStreams> Nx2SysmixStreamList() { + static const auto list = std::make_shared<Nx2SysmixStreams>(); + return list; +} + +// Switch 2 virtual output devices (nn::audio::ListVirtualOutputDevices and friends), reached through +// the object auddmg:u command 0 returns. Eden plays everything through one stereo 48 kHz output, so +// there is one virtual device. The nx2: name keeps the other commands auto-stubbed and logged. +// - 0(u32): u32 count, the device names in the out buffer (0x100 bytes each). +// - 100(u32): the default device's name in the out buffer. +// - 300(16-byte stream format, name buffer): whether the format is supported (result only). +// - 401(u32, name buffer): the device's 0x228-byte property block in the out buffer. The sdk hands +// the game the name, the channel count (0x108) and the u64 at 0x218; with the zeroed placeholder +// reply it saw 0 channels and Switch Sports failed to set up audio (2153-1604). +// - 500(16 bytes, name buffer): an event signalled when the device's state changes. +// - 700(u32 stream id, u32 device functionality, name buffer): attach a sysmix stream to the +// device, which starts it (nn::audio::devmgr::AttachStream, behind OutputStream Start). +// - 800(u32 stream id): detach it again (Stop). +class INx2AudioVirtualDeviceManager final : public ServiceFramework<INx2AudioVirtualDeviceManager> { +public: + explicit INx2AudioVirtualDeviceManager(Core::System& system_) + : ServiceFramework{system_, "nx2:auddmg:u:manager"}, + service_context{system_, "nx2:auddmg:u:manager"}, + state_event{service_context.CreateEvent("nx2:auddmg:u:StateEvent")} { + // clang-format off + static const FunctionInfo functions[] = { + {0, &INx2AudioVirtualDeviceManager::ListVirtualOutputDevices, "ListVirtualOutputDevices"}, + {100, &INx2AudioVirtualDeviceManager::GetDefaultVirtualOutputDevice, "GetDefaultVirtualOutputDevice"}, + {300, &INx2AudioVirtualDeviceManager::IsStreamFormatSupported, "IsVirtualOutputDeviceStreamFormatSupported"}, + {401, &INx2AudioVirtualDeviceManager::GetVirtualOutputDeviceProperties, "GetVirtualOutputDeviceProperties"}, + {500, &INx2AudioVirtualDeviceManager::AcquireStateChangeEvent, "AcquireVirtualOutputDeviceStateEvent (guessed)"}, + {700, &INx2AudioVirtualDeviceManager::AttachStream, "AttachStream"}, + {800, &INx2AudioVirtualDeviceManager::DetachStream, "DetachStream"}, + }; + // clang-format on + RegisterHandlers(functions); + } + + ~INx2AudioVirtualDeviceManager() override { + service_context.CloseEvent(state_event); + } + +private: + static constexpr std::size_t NameSize = 0x100; + static constexpr std::string_view DeviceName = "AudioVirtualOutput0"; + static constexpr std::string_view PhysicalDeviceName = "AudioTvOutput"; + + static void CopyName(u8* out, std::string_view name) { + std::memcpy(out, name.data(), std::min(name.size(), NameSize - 1)); + } + + void ListVirtualOutputDevices(HLERequestContext& ctx) { + std::array<u8, NameSize> names{}; + CopyName(names.data(), DeviceName); + const u32 count = ctx.GetWriteBufferSize() >= NameSize ? 1 : 0; + if (count) { + ctx.WriteBuffer(names); + } + LOG_DEBUG(Service_Audio, "called, {} device", count); + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(ResultSuccess); + rb.Push(count); + } + + void GetDefaultVirtualOutputDevice(HLERequestContext& ctx) { + std::array<u8, NameSize> name{}; + CopyName(name.data(), DeviceName); + ctx.WriteBuffer(name); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void IsStreamFormatSupported(HLERequestContext& ctx) { + LOG_DEBUG(Service_Audio, "called"); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + // Property block: name (0x000), u32 type (0x100, 3), u32 (0x104, 2), u32 channel count + // (0x108), u32 sample rate (0x10c), physical output name (0x110), its id (0x210), u64 latency + // in ns (0x218), u32 device id (0x220). The sdk's output streams pass the device id to + // sysmix CreateStream, which refuses 0 (Switch Sports failed with 2153-1604 on a zero there). + void GetVirtualOutputDeviceProperties(HLERequestContext& ctx) { + std::array<u8, 0x228> block{}; + const auto name = ctx.ReadBuffer(0); + std::memcpy(block.data(), name.data(), std::min(name.size(), NameSize - 1)); + const auto put32 = [&](std::size_t offset, u32 value) { + std::memcpy(block.data() + offset, &value, sizeof(value)); + }; + put32(0x100, 3); + put32(0x104, 2); + put32(0x108, 2); // stereo + put32(0x10c, 48000); // Hz + CopyName(block.data() + 0x110, PhysicalDeviceName); + const u64 latency_ns = 10'000'000; + std::memcpy(block.data() + 0x218, &latency_ns, sizeof(latency_ns)); + put32(0x220, 1); + ctx.WriteBuffer(block); + LOG_DEBUG(Service_Audio, "called for {}", Common::StringFromBuffer(name)); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void AcquireStateChangeEvent(HLERequestContext& ctx) { + IPC::ResponseBuilder rb{ctx, 2, 1}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, state_event->GetReadableEvent()); + } + + void AttachStream(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const auto stream_id = rp.Pop<u32>(); + const auto functionality = rp.Pop<u32>(); + LOG_INFO(Service_Audio, "attach stream {} to {} (functionality {})", stream_id, + Common::StringFromBuffer(ctx.ReadBuffer(0)), functionality); + SetStreamEngaged(stream_id, true); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void DetachStream(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const auto stream_id = rp.Pop<u32>(); + LOG_INFO(Service_Audio, "detach stream {}", stream_id); + SetStreamEngaged(stream_id, false); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void SetStreamEngaged(u32 stream_id, bool engaged) { + const auto list = Nx2SysmixStreamList(); + std::scoped_lock lk{list->mutex}; + if (const auto it = list->streams.find(stream_id); it != list->streams.end()) { + it->second->engaged = engaged; + it->second->owed_frames = 0; + if (it->second->state_event) { + it->second->state_event->Signal(system.Kernel()); + } + } + } + + KernelHelpers::ServiceContext service_context; + Kernel::KEvent* state_event; +}; + +// Switch 2 "auddmg:u": command 0 (16 bytes and the pid) returns the virtual device manager. +class INx2AudioDeviceManagerService final : public ServiceFramework<INx2AudioDeviceManagerService> { +public: + explicit INx2AudioDeviceManagerService(Core::System& system_) + : ServiceFramework{system_, "nx2:auddmg:u"} { + // clang-format off + static const FunctionInfo functions[] = { + {0, &INx2AudioDeviceManagerService::OpenVirtualDeviceManager, "OpenVirtualDeviceManager (guessed)"}, + }; + // clang-format on + RegisterHandlers(functions); + } + +private: + void OpenVirtualDeviceManager(HLERequestContext& ctx) { + LOG_DEBUG(Service_Audio, "called"); + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(ResultSuccess); + rb.PushIpcInterface<INx2AudioVirtualDeviceManager>(ctx, system); + } +}; + +// The stream object IUserSession command 1 returns. Commands (ids, request -> reply): +// - 0(u32 sample type, u32 channels, u32 sample rate, u32 frames, u32 prefill, u32 flags): open. +// - 1: close. 2: u32 state. 3: f32 gain. 4(f32 gain, s64 ramp ns): set the gain. +// - 5 / 6: acquire (copy handle) / release the state-change event. +// - 7(u32 mode, s32 threshold) / 8: acquire (copy handle) / release the data event. +// - 9(s64 ns): clock the stream. 10(s32 frames) -> u32: prime. 11(s32 frames) -> u32: purge. +// - 12(name buffer): rename. +class INx2SysmixStreamSession final : public ServiceFramework<INx2SysmixStreamSession> { +public: + explicit INx2SysmixStreamSession(Core::System& system_, std::shared_ptr<Nx2SysmixStreams> streams_, + std::shared_ptr<Nx2SysmixStream> stream_) + : ServiceFramework{system_, "nx2:audsmx:u:stream"}, service_context{system_, "nx2:audsmx:u:stream"}, + streams{std::move(streams_)}, stream{std::move(stream_)} { + // clang-format off + static const FunctionInfo functions[] = { + {0, &INx2SysmixStreamSession::OpenStream, "OpenStream"}, + {1, &INx2SysmixStreamSession::CloseStream, "CloseStream"}, + {2, &INx2SysmixStreamSession::GetStreamState, "GetStreamState"}, + {3, &INx2SysmixStreamSession::GetStreamGain, "GetStreamGain"}, + {4, &INx2SysmixStreamSession::SetStreamGain, "SetStreamGain"}, + {5, &INx2SysmixStreamSession::AcquireStateChangeEvent, "AcquireStreamStateChangeEvent"}, + {6, &INx2SysmixStreamSession::Succeed, "ReleaseStreamStateChangeEvent"}, + {7, &INx2SysmixStreamSession::AcquireDataEvent, "AcquireStreamDataEvent"}, + {8, &INx2SysmixStreamSession::Succeed, "ReleaseStreamDataEvent"}, + {9, &INx2SysmixStreamSession::Succeed, "ClockStream"}, + {10, &INx2SysmixStreamSession::EchoFrames, "PrimeStream"}, + {11, &INx2SysmixStreamSession::EchoFrames, "PurgeStream"}, + {12, &INx2SysmixStreamSession::Succeed, "RenameStream"}, + }; + // clang-format on + RegisterHandlers(functions); + std::scoped_lock lk{streams->mutex}; + stream->state_event = service_context.CreateEvent("nx2:audsmx:u:StreamStateEvent"); + stream->data_event = service_context.CreateEvent("nx2:audsmx:u:StreamDataEvent"); + } + + ~INx2SysmixStreamSession() override { + // The drain timer signals these events under the lock: unlist the stream first. + { + std::scoped_lock lk{streams->mutex}; + streams->streams.erase(stream->id); + service_context.CloseEvent(stream->state_event); + service_context.CloseEvent(stream->data_event); + stream->state_event = nullptr; + stream->data_event = nullptr; + } + } + +private: + // States the session reports: 0 created, 1 open, 2 open and engaged. + void OpenStream(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const auto sample_type = rp.Pop<u32>(); + const auto channels = rp.Pop<u32>(); + const auto sample_rate = rp.Pop<u32>(); + const auto frames = rp.Pop<u32>(); + const auto prefill = rp.Pop<u32>(); + const auto flags = rp.Pop<u32>(); + LOG_INFO(Service_Audio, + "stream {} open: sample type {}, {} channels, {} Hz, {} frames, prefill {}, flags {:#x}", + stream->id, sample_type, channels, sample_rate, frames, prefill, flags); + { + std::scoped_lock lk{streams->mutex}; + stream->sample_type = sample_type; + stream->channels = channels; + stream->sample_rate = sample_rate; + stream->frames = frames; + stream->owed_frames = 0; + stream->open = true; + stream->state_event->Signal(system.Kernel()); + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void CloseStream(HLERequestContext& ctx) { + { + std::scoped_lock lk{streams->mutex}; + stream->open = false; + stream->state_event->Signal(system.Kernel()); + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void GetStreamState(HLERequestContext& ctx) { + u32 state{}; + { + std::scoped_lock lk{streams->mutex}; + state = stream->open ? (stream->engaged ? 2 : 1) : 0; + } + LOG_DEBUG(Service_Audio, "stream {} state {}", stream->id, state); + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(ResultSuccess); + rb.Push(state); + } + + void GetStreamGain(HLERequestContext& ctx) { + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(ResultSuccess); + rb.Push(gain); + } + + void SetStreamGain(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + gain = rp.Pop<f32>(); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void AcquireStateChangeEvent(HLERequestContext& ctx) { + IPC::ResponseBuilder rb{ctx, 2, 1}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, stream->state_event->GetReadableEvent()); + } + + void AcquireDataEvent(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const auto mode = rp.Pop<u32>(); + const auto threshold = rp.Pop<s32>(); + LOG_INFO(Service_Audio, "stream {} data event: mode {}, threshold {}", stream->id, mode, threshold); + { + // The game waits on this before its first write, so signal it at once when the ring + // already has room (it is empty right after OpenStream); the drain timer keeps it up. + std::scoped_lock lk{streams->mutex}; + stream->data_mode = mode; + stream->data_threshold = threshold; + if (mode != 0 && stream->open && static_cast<s64>(stream->frames) >= threshold) { + stream->data_event->Signal(system.Kernel()); + } + } + IPC::ResponseBuilder rb{ctx, 2, 1}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, stream->data_event->GetReadableEvent()); + } + + void EchoFrames(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const auto frames = rp.Pop<s32>(); + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(ResultSuccess); + rb.Push<u32>(static_cast<u32>((std::max)(frames, 0))); + } + + void Succeed(HLERequestContext& ctx) { + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + KernelHelpers::ServiceContext service_context; + std::shared_ptr<Nx2SysmixStreams> streams; + std::shared_ptr<Nx2SysmixStream> stream; + f32 gain{1.0f}; +}; + +// The session audsmx:u command 0 returns. Commands: +// - 1(u32, u32 creation flags, u64 pid, u64 buffer address, u64 buffer size, name buffer): +// u32 stream id and the stream object. +// - 2 / 3(buffer of u32 stream ids): engage / disengage those streams. +class INx2SysmixUserSession final : public ServiceFramework<INx2SysmixUserSession> { +public: + explicit INx2SysmixUserSession(Core::System& system_) : ServiceFramework{system_, "nx2:audsmx:u:session"} { + // clang-format off + static const FunctionInfo functions[] = { + {1, &INx2SysmixUserSession::CreateStream, "CreateStream"}, + {2, &INx2SysmixUserSession::EngageStreams, "EngageStreams"}, + {3, &INx2SysmixUserSession::DisengageStreams, "DisengageStreams"}, + }; + // clang-format on + RegisterHandlers(functions); + drain_timer = Core::Timing::CreateEvent( + "nx2:audsmx:u drain", + [streams = streams, &system = system, owner = session_id](s64, std::chrono::nanoseconds) + -> std::optional<std::chrono::nanoseconds> { + Drain(system, *streams, owner); + return std::nullopt; + }); + system.CoreTiming().ScheduleLoopingEvent(DrainInterval, DrainInterval, drain_timer); + } + + ~INx2SysmixUserSession() override { + system.CoreTiming().UnscheduleEvent(drain_timer); + } + +private: + static constexpr std::chrono::nanoseconds DrainInterval{std::chrono::milliseconds{5}}; + static constexpr u64 ReadIndexOffset = 0x44; + static constexpr u64 WriteIndexOffset = 0x1004; + + // Takes up to one interval's worth of frames out of each engaged ring (frames owed while a ring + // is empty are dropped, as an underrun plays silence), and signals the data event of every open + // stream with at least its threshold of free frames, engaged or not: the game fills the ring + // before it engages the stream. + static void Drain(Core::System& system, Nx2SysmixStreams& state, u32 owner) { + std::scoped_lock lk{state.mutex}; + auto& memory = system.ApplicationMemory(); + for (auto& [id, stream] : state.streams) { + if (stream->owner != owner || !stream->open || stream->frames == 0 || !stream->buffer) { + continue; + } + const u32 capacity = stream->frames + 1; + const u32 write = memory.Read32(stream->buffer + WriteIndexOffset); + const u32 read = memory.Read32(stream->buffer + ReadIndexOffset); + if (write >= capacity || read >= capacity) { + continue; + } + u32 filled = (write + capacity - read) % capacity; + if (stream->engaged) { + stream->owed_frames += stream->sample_rate * + std::chrono::duration<double>(DrainInterval).count(); + const auto owed = static_cast<u32>(stream->owed_frames); + stream->owed_frames -= owed; + const u32 taken = (std::min)(filled, owed); + if (taken != 0) { + memory.Write32(stream->buffer + ReadIndexOffset, (read + taken) % capacity); + filled -= taken; + } + } + const u32 free_frames = capacity - 1 - filled; + if (stream->data_event && stream->data_mode != 0 && + static_cast<s64>(free_frames) >= stream->data_threshold) { + stream->data_event->Signal(system.Kernel()); + } + } + } + + void CreateStream(HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + const auto kind = rp.Pop<u32>(); + const auto flags = rp.Pop<u32>(); + [[maybe_unused]] const auto pid = rp.Pop<u64>(); + const auto buffer = rp.Pop<u64>(); + const auto size = rp.Pop<u64>(); + const auto name = Common::StringFromBuffer(ctx.ReadBuffer(0)); + auto stream = std::make_shared<Nx2SysmixStream>(); + stream->buffer = buffer; + stream->size = size; + { + std::scoped_lock lk{streams->mutex}; + stream->id = streams->next_id++; + stream->owner = session_id; + streams->streams.emplace(stream->id, stream); + } + LOG_INFO(Service_Audio, "stream {} '{}': kind {}, flags {:#x}, buffer {:#x} size {:#x}", + stream->id, name, kind, flags, buffer, size); + IPC::ResponseBuilder rb{ctx, 3, 0, 1}; + rb.Push(ResultSuccess); + rb.Push(stream->id); + rb.PushIpcInterface<INx2SysmixStreamSession>(ctx, system, streams, stream); + } + + void SetEngaged(HLERequestContext& ctx, bool engaged) { + const auto buffer = ctx.ReadBuffer(0); + std::vector<u32> ids(buffer.size() / sizeof(u32)); + std::memcpy(ids.data(), buffer.data(), ids.size() * sizeof(u32)); + { + std::scoped_lock lk{streams->mutex}; + for (const u32 id : ids) { + if (const auto it = streams->streams.find(id); it != streams->streams.end()) { + it->second->engaged = engaged; + it->second->owed_frames = 0; + if (it->second->state_event) { + it->second->state_event->Signal(system.Kernel()); + } + } + } + } + LOG_INFO(Service_Audio, "{} streams {}", engaged ? "engage" : "disengage", fmt::join(ids, ",")); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void EngageStreams(HLERequestContext& ctx) { + SetEngaged(ctx, true); + } + + void DisengageStreams(HLERequestContext& ctx) { + SetEngaged(ctx, false); + } + + static inline std::atomic<u32> next_session_id{1}; + const u32 session_id = next_session_id++; + std::shared_ptr<Nx2SysmixStreams> streams = Nx2SysmixStreamList(); + std::shared_ptr<Core::Timing::EventType> drain_timer; +}; + +// Switch 2 "audsmx:u": command 0 (16 bytes, the pid and a copy handle) returns the user session. +class INx2AudioSoundMixerService final : public ServiceFramework<INx2AudioSoundMixerService> { +public: + explicit INx2AudioSoundMixerService(Core::System& system_) + : ServiceFramework{system_, "nx2:audsmx:u"} { + // clang-format off + static const FunctionInfo functions[] = { + {0, &INx2AudioSoundMixerService::OpenUserSession, "OpenUserSession"}, + }; + // clang-format on + RegisterHandlers(functions); + } + +private: + void OpenUserSession(HLERequestContext& ctx) { + LOG_DEBUG(Service_Audio, "called"); + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(ResultSuccess); + rb.PushIpcInterface<INx2SysmixUserSession>(ctx, system); + } +}; + +} // namespace Service::Audio diff --git a/src/core/hle/service/caps/caps_su.cpp b/src/core/hle/service/caps/caps_su.cpp index 4ecfc840e4..b19e6b7a27 100644 --- a/src/core/hle/service/caps/caps_su.cpp +++ b/src/core/hle/service/caps/caps_su.cpp @@ -21,10 +21,10 @@ IScreenShotApplicationService::IScreenShotApplicationService( // clang-format off static const FunctionInfo functions[] = { {32, C<&IScreenShotApplicationService::SetShimLibraryVersion>, "SetShimLibraryVersion"}, - {201, nullptr, "SaveScreenShot"}, + {201, &IScreenShotApplicationService::StubReplyZeroed, "SaveScreenShot"}, {203, C<&IScreenShotApplicationService::SaveScreenShotEx0>, "SaveScreenShotEx0"}, {205, C<&IScreenShotApplicationService::SaveScreenShotEx1>, "SaveScreenShotEx1"}, - {210, nullptr, "SaveScreenShotEx2"}, + {210, &IScreenShotApplicationService::StubReplyZeroed, "SaveScreenShotEx2"}, }; // clang-format on diff --git a/src/core/hle/service/caps/caps_u.cpp b/src/core/hle/service/caps/caps_u.cpp index 6472b2bac6..eb4a10193e 100644 --- a/src/core/hle/service/caps/caps_u.cpp +++ b/src/core/hle/service/caps/caps_u.cpp @@ -20,17 +20,21 @@ IAlbumApplicationService::IAlbumApplicationService(Core::System& system_, static const FunctionInfo functions[] = { {32, C<&IAlbumApplicationService::SetShimLibraryVersion>, "SetShimLibraryVersion"}, {102, C<&IAlbumApplicationService::GetAlbumFileList0AafeAruidDeprecated>, "GetAlbumFileList0AafeAruidDeprecated"}, - {103, nullptr, "DeleteAlbumFileByAruid"}, - {104, nullptr, "GetAlbumFileSizeByAruid"}, - {105, nullptr, "DeleteAlbumFileByAruidForDebug"}, - {110, nullptr, "LoadAlbumScreenShotImageByAruid"}, - {120, nullptr, "LoadAlbumScreenShotThumbnailImageByAruid"}, - {130, nullptr, "PrecheckToCreateContentsByAruid"}, - {140, nullptr, "GetAlbumFileList1AafeAruidDeprecated"}, - {141, nullptr, "GetAlbumFileList2AafeUidAruidDeprecated"}, + {103, &IAlbumApplicationService::StubReplyZeroed, "DeleteAlbumFileByAruid"}, + {104, &IAlbumApplicationService::StubReplyZeroed, "GetAlbumFileSizeByAruid"}, + {105, &IAlbumApplicationService::StubReplyZeroed, "DeleteAlbumFileByAruidForDebug"}, + {110, &IAlbumApplicationService::StubReplyZeroed, "LoadAlbumScreenShotImageByAruid"}, + {120, &IAlbumApplicationService::StubReplyZeroed, "LoadAlbumScreenShotThumbnailImageByAruid"}, + {130, &IAlbumApplicationService::StubReplyZeroed, "PrecheckToCreateContentsByAruid"}, + {140, &IAlbumApplicationService::StubReplyZeroed, "GetAlbumFileList1AafeAruidDeprecated"}, + {141, &IAlbumApplicationService::StubReplyZeroed, "GetAlbumFileList2AafeUidAruidDeprecated"}, {142, C<&IAlbumApplicationService::GetAlbumFileList3AaeAruid>, "GetAlbumFileList3AaeAruid"}, - {143, nullptr, "GetAlbumFileList4AaeUidAruid"}, - {144, nullptr, "GetAllAlbumFileList3AaeAruid"}, + {143, &IAlbumApplicationService::StubReplyZeroed, "GetAlbumFileList4AaeUidAruid"}, + {144, &IAlbumApplicationService::StubReplyZeroed, "GetAllAlbumFileList3AaeAruid"}, + {145, &IAlbumApplicationService::StubReplyZeroed, "Unknown145"}, + {146, &IAlbumApplicationService::StubReplyZeroed, "Unknown146"}, + {147, &IAlbumApplicationService::StubReplyZeroed, "Unknown147"}, + {148, &IAlbumApplicationService::StubReplyZeroed, "Unknown148"}, {60002, nullptr, "OpenAccessorSessionForApplication"}, }; // clang-format on diff --git a/src/core/hle/service/hid/hid_server.cpp b/src/core/hle/service/hid/hid_server.cpp index 04b4663bc2..20118345a9 100644 --- a/src/core/hle/service/hid/hid_server.cpp +++ b/src/core/hle/service/hid/hid_server.cpp @@ -74,12 +74,12 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {70, C<&IHidServer::SetSixAxisSensorFusionParameters>, "SetSixAxisSensorFusionParameters"}, {71, C<&IHidServer::GetSixAxisSensorFusionParameters>, "GetSixAxisSensorFusionParameters"}, {72, C<&IHidServer::ResetSixAxisSensorFusionParameters>, "ResetSixAxisSensorFusionParameters"}, - {73, nullptr, "SetAccelerometerParameters"}, - {74, nullptr, "GetAccelerometerParameters"}, - {75, nullptr, "ResetAccelerometerParameters"}, - {76, nullptr, "SetAccelerometerPlayMode"}, - {77, nullptr, "GetAccelerometerPlayMode"}, - {78, nullptr, "ResetAccelerometerPlayMode"}, + {73, &IHidServer::StubReplyZeroed, "SetAccelerometerParameters"}, + {74, &IHidServer::StubReplyZeroed, "GetAccelerometerParameters"}, + {75, &IHidServer::StubReplyZeroed, "ResetAccelerometerParameters"}, + {76, &IHidServer::StubReplyZeroed, "SetAccelerometerPlayMode"}, + {77, &IHidServer::StubReplyZeroed, "GetAccelerometerPlayMode"}, + {78, &IHidServer::StubReplyZeroed, "ResetAccelerometerPlayMode"}, {79, C<&IHidServer::SetGyroscopeZeroDriftMode>, "SetGyroscopeZeroDriftMode"}, {80, C<&IHidServer::GetGyroscopeZeroDriftMode>, "GetGyroscopeZeroDriftMode"}, {81, C<&IHidServer::ResetGyroscopeZeroDriftMode>, "ResetGyroscopeZeroDriftMode"}, @@ -87,7 +87,7 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {83, C<&IHidServer::IsFirmwareUpdateAvailableForSixAxisSensor>, "IsFirmwareUpdateAvailableForSixAxisSensor"}, {84, C<&IHidServer::EnableSixAxisSensorUnalteredPassthrough>, "EnableSixAxisSensorUnalteredPassthrough"}, {85, C<&IHidServer::IsSixAxisSensorUnalteredPassthroughEnabled>, "IsSixAxisSensorUnalteredPassthroughEnabled"}, - {86, nullptr, "StoreSixAxisSensorCalibrationParameter"}, + {86, &IHidServer::StubReplyZeroed, "StoreSixAxisSensorCalibrationParameter"}, {87, C<&IHidServer::LoadSixAxisSensorCalibrationParameter>, "LoadSixAxisSensorCalibrationParameter"}, {88, C<&IHidServer::GetSixAxisSensorIcInformation>, "GetSixAxisSensorIcInformation"}, {89, C<&IHidServer::ResetIsSixAxisSensorDeviceNewlyAssigned>, "ResetIsSixAxisSensorDeviceNewlyAssigned"}, @@ -119,6 +119,13 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {134, C<&IHidServer::SetNpadAnalogStickUseCenterClamp>, "SetNpadAnalogStickUseCenterClamp"}, {135, C<&IHidServer::SetNpadCaptureButtonAssignment>, "SetNpadCaptureButtonAssignment"}, {136, C<&IHidServer::ClearNpadCaptureButtonAssignment>, "ClearNpadCaptureButtonAssignment"}, + {137, &IHidServer::StubReplyZeroed, "Unknown137"}, + {138, &IHidServer::StubReplyZeroed, "Unknown138"}, + {139, &IHidServer::StubReplyZeroed, "Unknown139"}, + {140, &IHidServer::StubReplyZeroed, "Unknown140"}, + {150, &IHidServer::StubReplyZeroed, "Unknown150"}, + {151, &IHidServer::StubReplyZeroed, "Unknown151"}, + {160, &IHidServer::StubReplyZeroed, "Unknown160"}, {200, C<&IHidServer::GetVibrationDeviceInfo>, "GetVibrationDeviceInfo"}, {201, C<&IHidServer::SendVibrationValue>, "SendVibrationValue"}, {202, C<&IHidServer::GetActualVibrationValue>, "GetActualVibrationValue"}, @@ -132,7 +139,9 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {210, C<&IHidServer::EndPermitVibrationSession>, "EndPermitVibrationSession"}, {211, C<&IHidServer::IsVibrationDeviceMounted>, "IsVibrationDeviceMounted"}, {212, C<&IHidServer::SendVibrationValueInBool>, "SendVibrationValueInBool"}, + {213, C<&IHidServer::Unknown213>, "Unknown213"}, {214, C<&IHidServer::SendNx2VibrationValues>, "SendNx2VibrationValues (guessed)"}, + {220, C<&IHidServer::Unknown220>, "Unknown220"}, {300, C<&IHidServer::ActivateConsoleSixAxisSensor>, "ActivateConsoleSixAxisSensor"}, {301, C<&IHidServer::StartConsoleSixAxisSensor>, "StartConsoleSixAxisSensor"}, {302, C<&IHidServer::StopConsoleSixAxisSensor>, "StopConsoleSixAxisSensor"}, @@ -141,19 +150,24 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {305, C<&IHidServer::StopSevenSixAxisSensor>, "StopSevenSixAxisSensor"}, {306, C<&IHidServer::InitializeSevenSixAxisSensor>, "InitializeSevenSixAxisSensor"}, {307, C<&IHidServer::FinalizeSevenSixAxisSensor>, "FinalizeSevenSixAxisSensor"}, - {308, nullptr, "SetSevenSixAxisSensorFusionStrength"}, - {309, nullptr, "GetSevenSixAxisSensorFusionStrength"}, + {308, &IHidServer::StubReplyZeroed, "SetSevenSixAxisSensorFusionStrength"}, + {309, &IHidServer::StubReplyZeroed, "GetSevenSixAxisSensorFusionStrength"}, {310, C<&IHidServer::ResetSevenSixAxisSensorTimestamp>, "ResetSevenSixAxisSensorTimestamp"}, - {320, nullptr, "EnableNpadImu"}, //21.0.0+ - {321, nullptr, "DisableNpadImu"}, //21.0.0+ + {311, &IHidServer::StubReplyZeroed, "Unknown311"}, + {312, &IHidServer::StubReplyZeroed, "Unknown312"}, + {320, &IHidServer::StubReplyZeroed, "EnableNpadImu"}, //21.0.0+ + {321, &IHidServer::StubReplyZeroed, "DisableNpadImu"}, //21.0.0+ + {322, &IHidServer::StubReplyZeroed, "Unknown322"}, + {323, &IHidServer::StubReplyZeroed, "Unknown323"}, + {324, &IHidServer::StubReplyZeroed, "Unknown324"}, {400, C<&IHidServer::IsUsbFullKeyControllerEnabled>, "IsUsbFullKeyControllerEnabled"}, - {401, nullptr, "EnableUsbFullKeyController"}, - {402, nullptr, "IsUsbFullKeyControllerConnected"}, - {403, nullptr, "HasBattery"}, - {404, nullptr, "HasLeftRightBattery"}, - {405, nullptr, "GetNpadInterfaceType"}, - {406, nullptr, "GetNpadLeftRightInterfaceType"}, - {407, nullptr, "GetNpadOfHighestBatteryLevel"}, + {401, &IHidServer::StubReplyZeroed, "EnableUsbFullKeyController"}, + {402, &IHidServer::StubReplyZeroed, "IsUsbFullKeyControllerConnected"}, + {403, &IHidServer::StubReplyZeroed, "HasBattery"}, + {404, &IHidServer::StubReplyZeroed, "HasLeftRightBattery"}, + {405, &IHidServer::StubReplyZeroed, "GetNpadInterfaceType"}, + {406, &IHidServer::StubReplyZeroed, "GetNpadLeftRightInterfaceType"}, + {407, &IHidServer::StubReplyZeroed, "GetNpadOfHighestBatteryLevel"}, {408, nullptr, "GetNpadOfHighestBatteryLevelForJoyRight"}, {500, C<&IHidServer::GetPalmaConnectionHandle>, "GetPalmaConnectionHandle"}, {501, C<&IHidServer::InitializePalma>, "InitializePalma"}, @@ -185,11 +199,13 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {527, C<&IHidServer::EnablePalmaBoostMode>, "EnablePalmaBoostMode"}, {528, C<&IHidServer::GetPalmaBluetoothAddress>, "GetPalmaBluetoothAddress"}, {529, C<&IHidServer::SetDisallowedPalmaConnection>, "SetDisallowedPalmaConnection"}, + {550, &IHidServer::StubReplyZeroed, "Unknown550"}, {1000, C<&IHidServer::SetNpadCommunicationMode>, "SetNpadCommunicationMode"}, {1001, C<&IHidServer::GetNpadCommunicationMode>, "GetNpadCommunicationMode"}, {1002, C<&IHidServer::SetTouchScreenConfiguration>, "SetTouchScreenConfiguration"}, {1003, C<&IHidServer::IsFirmwareUpdateNeededForNotification>, "IsFirmwareUpdateNeededForNotification"}, {1004, C<&IHidServer::SetTouchScreenResolution>, "SetTouchScreenResolution"}, + {1005, &IHidServer::StubReplyZeroed, "Unknown1005"}, {1270, nullptr, "DeleteButtonConfigStorageRight"}, {1271, nullptr, "IsUsingCustomButtonConfig"}, {1272, nullptr, "IsAnyCustomButtonConfigEnabled"}, @@ -222,23 +238,49 @@ IHidServer::IHidServer(Core::System& system_, std::shared_ptr<ResourceManager> r {1325, nullptr, "IsTouchScreenAutoTuneRequiredForRepairProviderReplacement"}, {1420, nullptr, "GetAppletResourceProperty"}, {2000, nullptr, "ActivateDigitizer"}, - {3000, nullptr, "GetDebugPadGenericPadMap"}, - {3001, nullptr, "SetDebugPadGenericPadMap"}, - {3002, nullptr, "ResetDebugPadGenericPadMap"}, - {3003, nullptr, "GetDebugPadKeyboardMap"}, - {3004, nullptr, "SetDebugPadKeyboardMap"}, - {3005, nullptr, "ResetDebugPadKeyboardMap"}, - {3006, nullptr, "GetFullKeyGenericPadMap"}, - {3007, nullptr, "SetFullKeyGenericPadMap"}, - {3008, nullptr, "ResetFullKeyGenericPadMap"}, - {3009, nullptr, "GetFullKeyKeyboardMap"}, - {3010, nullptr, "SetFullKeyKeyboardMap"}, - {3011, nullptr, "ResetFullKeyKeyboardMap"}, - {3012, nullptr, "GetDebugPadGenericPadMap"}, //21.0.0+ - {3013, nullptr, "SetDebugPadGenericPadMap"}, //21.0.0+ - {3014, nullptr, "GetDebugPadKeyboardMap"}, //21.0.0+ - {3015, nullptr, "SetDebugPadKeyboardMap"}, //21.0.0+ + {3000, &IHidServer::StubReplyZeroed, "GetDebugPadGenericPadMap"}, + {3001, &IHidServer::StubReplyZeroed, "SetDebugPadGenericPadMap"}, + {3002, &IHidServer::StubReplyZeroed, "ResetDebugPadGenericPadMap"}, + {3003, &IHidServer::StubReplyZeroed, "GetDebugPadKeyboardMap"}, + {3004, &IHidServer::StubReplyZeroed, "SetDebugPadKeyboardMap"}, + {3005, &IHidServer::StubReplyZeroed, "ResetDebugPadKeyboardMap"}, + {3006, &IHidServer::StubReplyZeroed, "GetFullKeyGenericPadMap"}, + {3007, &IHidServer::StubReplyZeroed, "SetFullKeyGenericPadMap"}, + {3008, &IHidServer::StubReplyZeroed, "ResetFullKeyGenericPadMap"}, + {3009, &IHidServer::StubReplyZeroed, "GetFullKeyKeyboardMap"}, + {3010, &IHidServer::StubReplyZeroed, "SetFullKeyKeyboardMap"}, + {3011, &IHidServer::StubReplyZeroed, "ResetFullKeyKeyboardMap"}, + {3012, &IHidServer::StubReplyZeroed, "GetDebugPadGenericPadMap"}, //21.0.0+ + {3013, &IHidServer::StubReplyZeroed, "SetDebugPadGenericPadMap"}, //21.0.0+ + {3014, &IHidServer::StubReplyZeroed, "GetDebugPadKeyboardMap"}, //21.0.0+ + {3015, &IHidServer::StubReplyZeroed, "SetDebugPadKeyboardMap"}, //21.0.0+ + {3110, &IHidServer::StubReplyZeroed, "Unknown3110"}, + {3111, &IHidServer::StubReplyZeroed, "Unknown3111"}, + {3112, &IHidServer::StubReplyZeroed, "Unknown3112"}, + {3113, &IHidServer::StubReplyZeroed, "Unknown3113"}, + {3114, &IHidServer::StubReplyZeroed, "Unknown3114"}, + {3130, &IHidServer::StubReplyZeroed, "Unknown3130"}, + {3131, &IHidServer::StubReplyZeroed, "Unknown3131"}, + {3132, &IHidServer::StubReplyZeroed, "Unknown3132"}, + {3133, &IHidServer::StubReplyZeroed, "Unknown3133"}, + {3134, &IHidServer::StubReplyZeroed, "Unknown3134"}, + {3135, &IHidServer::StubReplyZeroed, "Unknown3135"}, + {3136, &IHidServer::StubReplyZeroed, "Unknown3136"}, + {3137, &IHidServer::StubReplyZeroed, "Unknown3137"}, + {3138, &IHidServer::StubReplyZeroed, "Unknown3138"}, + {3139, &IHidServer::StubReplyZeroed, "Unknown3139"}, + {3140, &IHidServer::StubReplyZeroed, "Unknown3140"}, + {3141, &IHidServer::StubReplyZeroed, "Unknown3141"}, + {3142, &IHidServer::StubReplyZeroed, "Unknown3142"}, + {3143, &IHidServer::StubReplyZeroed, "Unknown3143"}, + {3144, &IHidServer::StubReplyZeroed, "Unknown3144"}, + {3145, &IHidServer::StubReplyZeroed, "Unknown3145"}, + {3146, &IHidServer::StubReplyZeroed, "Unknown3146"}, + {3147, &IHidServer::StubReplyZeroed, "Unknown3147"}, + {3148, &IHidServer::StubReplyZeroed, "Unknown3148"}, + {3149, &IHidServer::StubReplyZeroed, "Unknown3149"}, {3150, C<&IHidServer::SetMouseLibraryVersion>, "SetMouseLibraryVersion"}, //21.0.0+ + {5000, &IHidServer::StubReplyZeroed, "Unknown5000"}, // What? -- {12010, nullptr, "SetButtonConfigLeft"}, }; // clang-format on @@ -1075,6 +1117,17 @@ Result IHidServer::SendNx2VibrationValues() { R_SUCCEED(); } +Result IHidServer::Unknown213() { + LOG_DEBUG(Service_HID, "(STUBBED) called"); + R_SUCCEED(); +} + +Result IHidServer::Unknown220(Out<u8> out) { + LOG_DEBUG(Service_HID, "(STUBBED) called"); + *out = 0; + R_SUCCEED(); +} + Result IHidServer::SendVibrationValueInBool( bool is_vibrating, Core::HID::VibrationDeviceHandle vibration_device_handle, ClientAppletResourceUserId aruid) { diff --git a/src/core/hle/service/hid/hid_server.h b/src/core/hle/service/hid/hid_server.h index 574d54c415..0767bb91b7 100644 --- a/src/core/hle/service/hid/hid_server.h +++ b/src/core/hle/service/hid/hid_server.h @@ -192,6 +192,12 @@ private: // Switch 2 sends vibration every frame with 214 (pointer buffers of device handles and // values, like 206); Eden doesn't emulate vibration, so accept and drop it. Result SendNx2VibrationValues(); + // 213 (32 bytes, aruid): sent per controller every frame by Kirby Air Riders (tens of + // thousands of auto stub log lines a run); no reply data. Accepted and dropped like 214. + Result Unknown213(); + // 220 (16 bytes, aruid): replies one byte (meaning unknown); answer 0 instead of leaving it to + // the message buffer's leftovers. + Result Unknown220(Out<u8> out); Result ActivateConsoleSixAxisSensor(ClientAppletResourceUserId aruid); Result StartConsoleSixAxisSensor(Core::HID::ConsoleSixAxisSensorHandle console_sixaxis_handle, ClientAppletResourceUserId aruid); diff --git a/src/core/hle/service/kernel_helpers.cpp b/src/core/hle/service/kernel_helpers.cpp index e4d6925f19..db55c6f81a 100644 --- a/src/core/hle/service/kernel_helpers.cpp +++ b/src/core/hle/service/kernel_helpers.cpp @@ -48,6 +48,17 @@ ServiceContext::~ServiceContext() { } } +namespace { +std::mutex debug_names_mutex; +std::unordered_map<const Kernel::KReadableEvent*, std::string> debug_names; +} // Anonymous namespace + +std::string DebugEventName(const Kernel::KReadableEvent* event) { + std::scoped_lock lk{debug_names_mutex}; + const auto it = debug_names.find(event); + return it != debug_names.end() ? it->second : std::string{"?"}; +} + Kernel::KEvent* ServiceContext::CreateEvent(std::string&& name) { // Reserve a new event from the process resource limit Kernel::KScopedResourceReservation event_reservation(kernel, process, @@ -74,6 +85,10 @@ Kernel::KEvent* ServiceContext::CreateEvent(std::string&& name) { Kernel::KEvent::Register(kernel, event); { + std::scoped_lock lk{debug_names_mutex}; + debug_names[&event->GetReadableEvent()] = name; + } + { std::scoped_lock lk{event_names_mutex}; event_names[event] = std::move(name); } @@ -86,6 +101,10 @@ void ServiceContext::CloseEvent(Kernel::KEvent* event) { std::scoped_lock lk{event_names_mutex}; event_names.erase(event); } + { + std::scoped_lock lk{debug_names_mutex}; + debug_names.erase(&event->GetReadableEvent()); + } event->GetReadableEvent().Close(kernel); event->Close(kernel); } diff --git a/src/core/hle/service/kernel_helpers.h b/src/core/hle/service/kernel_helpers.h index a62dd838aa..b03e49300f 100644 --- a/src/core/hle/service/kernel_helpers.h +++ b/src/core/hle/service/kernel_helpers.h @@ -16,6 +16,7 @@ namespace Kernel { class KernelCore; class KEvent; class KProcess; +class KReadableEvent; } // namespace Kernel namespace Service::KernelHelpers { @@ -37,4 +38,7 @@ public: bool process_created{false}; }; +/// The name a service gave an event (EDEN_NX2_SVC_TRACE shows the handles a thread waits on). +std::string DebugEventName(const Kernel::KReadableEvent* event); + } // namespace Service::KernelHelpers diff --git a/src/core/hle/service/lm/lm.cpp b/src/core/hle/service/lm/lm.cpp index d631a6dd80..a7d6d6e707 100644 --- a/src/core/hle/service/lm/lm.cpp +++ b/src/core/hle/service/lm/lm.cpp @@ -91,8 +91,9 @@ public: static const FunctionInfo functions[] = { {0, &ILogger::Log, "Log"}, {1, &ILogger::SetDestination, "SetDestination"}, - {2, nullptr, "TransmitHashedLog"}, //20.0.0+ - {3, nullptr, "DevNotify"}, //20.0.0+ + {2, &ILogger::StubReplyZeroed, "TransmitHashedLog"}, //20.0.0+ + {3, &ILogger::StubReplyZeroed, "DevNotify"}, //20.0.0+ + {11, &ILogger::StubReplyInterface, "Unknown11"}, }; RegisterHandlers(functions); } diff --git a/src/core/hle/service/mii/mii.cpp b/src/core/hle/service/mii/mii.cpp index 513bb671f8..ffdb17fd14 100644 --- a/src/core/hle/service/mii/mii.cpp +++ b/src/core/hle/service/mii/mii.cpp @@ -48,8 +48,8 @@ public: {15, D<&IDatabaseService::DestroyFile>, "DestroyFile"}, {16, D<&IDatabaseService::DeleteFile>, "DeleteFile"}, {17, D<&IDatabaseService::Format>, "Format"}, - {18, nullptr, "Import"}, - {19, nullptr, "Export"}, + {18, &IDatabaseService::StubReplyZeroed, "Import"}, + {19, &IDatabaseService::StubReplyZeroed, "Export"}, {20, D<&IDatabaseService::IsBrokenDatabaseWithClearFlag>, "IsBrokenDatabaseWithClearFlag"}, {21, D<&IDatabaseService::GetIndex>, "GetIndex"}, {22, D<&IDatabaseService::SetInterfaceVersion>, "SetInterfaceVersion"}, @@ -57,6 +57,8 @@ public: {24, D<&IDatabaseService::ConvertCoreDataToCharInfo>, "ConvertCoreDataToCharInfo"}, {25, D<&IDatabaseService::ConvertCharInfoToCoreData>, "ConvertCharInfoToCoreData"}, {26, D<&IDatabaseService::Append>, "Append"}, + {27, &IDatabaseService::StubReplyZeroed, "Unknown27"}, + {28, &IDatabaseService::StubReplyZeroed, "Unknown28"}, }; // clang-format on diff --git a/src/core/hle/service/ns/platform_service_manager.cpp b/src/core/hle/service/ns/platform_service_manager.cpp index 88aac56b1c..7bd8980a08 100644 --- a/src/core/hle/service/ns/platform_service_manager.cpp +++ b/src/core/hle/service/ns/platform_service_manager.cpp @@ -85,8 +85,12 @@ struct IPlatformServiceManager::Impl { // Automatically populated based on shared_fonts dump or system archives. // 6 builtin fonts + extra 2 for whatever may come after boost::container::static_vector<FontRegion, 8> shared_font_regions; - /// Plain TTF copies of the same fonts (Switch 2 GetBuiltInFontData), after the BFTTF data. + /// Where Switch 2 GetBuiltInFontData finds the same fonts as plain TTF data: page-aligned + /// copies in builtin_font_memory (the Switch 1 font memory is full with the system fonts). boost::container::static_vector<FontRegion, 8> builtin_font_regions; + u32 builtin_font_memory_size{}; + /// Made by the first GetBuiltInFontSharedMemory (Switch 2 titles only). + Kernel::KSharedMemory* builtin_font_memory{}; /// Backing memory for the shared font data std::array<u8, SHARED_FONT_MEM_SIZE> shared_font; }; @@ -122,7 +126,6 @@ IPlatformServiceManager::IPlatformServiceManager(Core::System& system_, const ch // Attempt to load shared font data from disk const auto* nand = fsc.GetSystemNANDContents(); std::size_t offset = 0; - std::vector<std::vector<u8>> plain_fonts; // Rebuild shared fonts from data ncas or synthesize for (auto& font : SHARED_FONTS) { FileSys::VirtualFile romfs; @@ -158,34 +161,52 @@ IPlatformServiceManager::IPlatformServiceManager(Core::System& system_, const ch Common::swap32); // Font offset and size do not account for the header const FontRegion region{u32(offset + 8), u32((font_data_u32.size() * sizeof(u32)) - 8)}; - std::vector<u8> plain(region.size); - DecryptSharedFontToTTF(font_data_u32, plain); - plain_fonts.push_back(std::move(plain)); DecryptSharedFont(font_data_u32, impl->shared_font, offset); impl->shared_font_regions.push_back(region); } - for (const auto& plain : plain_fonts) { - offset = Common::AlignUp(offset, 0x1000); - if (offset + plain.size() > SHARED_FONT_MEM_SIZE) { - LOG_WARNING(Service_NS, "No shared memory left for plain built-in font copies"); - break; - } - std::memcpy(impl->shared_font.data() + offset, plain.data(), plain.size()); - impl->builtin_font_regions.push_back({u32(offset), u32(plain.size())}); - offset += plain.size(); + // The data after each BFTTF header is the plain font: GetBuiltInFontSharedMemory copies it. + u32 builtin_offset = 0; + for (const auto& region : impl->shared_font_regions) { + impl->builtin_font_regions.push_back({builtin_offset, region.size}); + builtin_offset = Common::AlignUp(builtin_offset + region.size, 0x1000); } + impl->builtin_font_memory_size = std::max<u32>(builtin_offset, 0x1000); } // Switch 2: the shared memory GetBuiltInFontData offsets point into (copy handle, u32 size); the -// SDK maps it read-only. Eden serves its font shared memory, which holds plain copies too. +// SDK maps it read-only. It holds plain copies of the fonts, apart from the Switch 1 font memory. void IPlatformServiceManager::GetBuiltInFontSharedMemory(HLERequestContext& ctx) { - LOG_WARNING(Service_NS, "(STUBBED) Switch 2 GetBuiltInFontSharedMemory"); - std::memcpy(kernel.GetFontSharedMem().GetPointer(), impl->shared_font.data(), impl->shared_font.size()); + LOG_WARNING(Service_NS, "(STUBBED) Switch 2 GetBuiltInFontSharedMemory, {:#x} bytes", + impl->builtin_font_memory_size); + if (impl->builtin_font_memory == nullptr) { + auto* const memory = Kernel::KSharedMemory::Create(kernel); + if (const Result result = memory->Initialize(kernel, system.DeviceMemory(), nullptr, + Kernel::Svc::MemoryPermission::None, + Kernel::Svc::MemoryPermission::Read, + impl->builtin_font_memory_size); + result.IsError()) { + LOG_ERROR(Service_NS, "Failed to make {:#x} bytes of built-in font memory: {:#x}", + impl->builtin_font_memory_size, result.raw); + memory->Close(kernel); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(result); + return; + } + Kernel::KSharedMemory::Register(kernel, memory); + u8* const data = memory->GetPointer(); + std::memset(data, 0, impl->builtin_font_memory_size); + for (size_t i = 0; i < impl->builtin_font_regions.size(); ++i) { + std::memcpy(data + impl->builtin_font_regions[i].offset, + impl->shared_font.data() + impl->shared_font_regions[i].offset, + impl->builtin_font_regions[i].size); + } + impl->builtin_font_memory = memory; + } IPC::ResponseBuilder rb{ctx, 3, 1}; rb.Push(ResultSuccess); - rb.Push<u32>(static_cast<u32>(SHARED_FONT_MEM_SIZE)); - rb.PushCopyObjects(ctx, kernel.GetFontSharedMem()); + rb.Push<u32>(impl->builtin_font_memory_size); + rb.PushCopyObjects(ctx, *impl->builtin_font_memory); } // nn::pl::GetBuiltInFontData(BuiltInFontDataOunce*, BuiltInFontTypeOunce, bool): returns a 16-byte @@ -212,7 +233,11 @@ void IPlatformServiceManager::GetBuiltInFontData(HLERequestContext& ctx) { rb.Push<u32>(0); } -IPlatformServiceManager::~IPlatformServiceManager() = default; +IPlatformServiceManager::~IPlatformServiceManager() { + if (impl->builtin_font_memory != nullptr) { + impl->builtin_font_memory->Close(kernel); + } +} Result IPlatformServiceManager::RequestLoad(SharedFontType type) { // Games don't call this so all fonts should be loaded diff --git a/src/core/hle/service/nvdrv/core/container.cpp b/src/core/hle/service/nvdrv/core/container.cpp index 9edce03f64..22db243e3f 100644 --- a/src/core/hle/service/nvdrv/core/container.cpp +++ b/src/core/hle/service/nvdrv/core/container.cpp @@ -11,6 +11,7 @@ #include "core/hle/service/nvdrv/core/heap_mapper.h" #include "core/hle/service/nvdrv/core/nvmap.h" #include "core/hle/service/nvdrv/core/syncpoint_manager.h" +#include "core/hle/service/nvdrv/devices/nx2_stub_device.h" #include "core/memory.h" #include "video_core/host1x/host1x.h" @@ -130,6 +131,7 @@ void Container::CloseSession(SessionId session_id) { session.has_preallocated_area = false; } session.is_active = false; + Devices::Nx2GpuMemoryRegistry::Get().ForgetDeviceRanges(session.asid.id); smmu.UnregisterProcess(impl->sessions[session_id.id].asid); impl->id_pool.emplace_front(session_id.id); } diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp index e66c72cd26..40313a10fb 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp @@ -5,6 +5,7 @@ // SPDX-FileCopyrightText: 2021 Skyline Team and Contributors // SPDX-License-Identifier: GPL-3.0-or-later +#include <cstdlib> #include <cstring> #include <utility> @@ -35,6 +36,13 @@ nvhost_as_gpu::nvhost_as_gpu(Core::System& system_, Module& module_, NvCore::Con nvhost_as_gpu::~nvhost_as_gpu() { Nx2GpuMemoryRegistry::Get().RemoveListener(this); + // Leave shared ranges to the process's other address spaces; like this address space's own + // ranges, one nobody else maps stays allocated. + for (const auto& [key, range] : nx2_device_ranges) { + if (range.shared) { + Nx2GpuMemoryRegistry::Get().ReleaseDeviceRange(range.asid, key.first, key.second); + } + } } NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input, @@ -134,27 +142,43 @@ void nvhost_as_gpu::Nx2Map(u64 gpu_address, u64 cpu_address, u64 size) { } // The same CPU range is often mapped twice (at its own address and in a sparse region), so // reuse its SMMU mapping. - DAddr device_address{}; - bool heap = false; + Nx2DeviceRange range{}; if (const auto it = nx2_device_ranges.find({cpu_address, size}); it != nx2_device_ranges.end()) { - device_address = it->second.address; - heap = it->second.heap; + range = it->second; } else if (session->has_preallocated_area && session->mapper->IsInBounds(cpu_address, size)) { - device_address = session->mapper->Map(cpu_address, size); - heap = true; + range = {session->mapper->Map(cpu_address, size), true}; } else { - auto& smmu = system.Host1x().MemoryManager(); - device_address = smmu.Allocate(Common::AlignUp(size, 0x20000)); - if (device_address == 0) { + const auto allocate = [&]() -> u64 { + auto& smmu = system.Host1x().MemoryManager(); + const DAddr address = smmu.Allocate(Common::AlignUp(size, 0x20000)); + if (address != 0) { + smmu.Map(address, cpu_address, size, session->asid, true); + } + return address; + }; + // EDEN_NX2_PRIVATE_DEVICE_RANGES=1 (A/B aid): a range per address space. + static const bool private_ranges = std::getenv("EDEN_NX2_PRIVATE_DEVICE_RANGES") != nullptr; + if (private_ranges) { + range = {allocate(), false}; + } else { + range = {Nx2GpuMemoryRegistry::Get().AcquireDeviceRange(session->asid.id, cpu_address, + size, allocate), + false, true, session->asid.id}; + } + if (range.address == 0) { LOG_CRITICAL(Service_NVDRV, "Switch 2 GPU memory {:#x}+{:#x}: out of SMMU space", cpu_address, size); return; } - smmu.Map(device_address, cpu_address, size, session->asid, true); } - nx2_device_ranges[{cpu_address, size}] = {device_address, heap}; + const DAddr device_address = range.address; + nx2_device_ranges[{cpu_address, size}] = range; Nx2GpuMemoryRegistry::Get().RecordDeviceRange(cpu_address, size, device_address); - gmmu->Map(gpu_address, device_address, size, Tegra::PTEKind::PITCH, false); + // On the GPU thread, like unmaps: a GMMU map takes the texture cache lock, and the callers hold + // nx2:gpu locks that the GPU thread takes under that lock (sparse blocks backed on write). + system.GPU().RunInOrder([gmmu = gmmu.get(), gpu_address, device_address, size] { + gmmu->Map(gpu_address, device_address, size, Tegra::PTEKind::PITCH, false); + }); LOG_DEBUG(Service_NVDRV, "Switch 2 GPU mapping {:#x}+{:#x} -> CPU {:#x}", gpu_address, size, cpu_address); } @@ -177,6 +201,10 @@ void nvhost_as_gpu::Nx2Release(u64 cpu_address, u64 size) { } const Nx2DeviceRange range = it->second; nx2_device_ranges.erase(it); + if (range.shared && + !Nx2GpuMemoryRegistry::Get().ReleaseDeviceRange(range.asid, cpu_address, size)) { + return; // another address space of the process still maps it + } auto* session = container.GetSession(nx2_session); auto* const mapper = range.heap && session != nullptr ? session->mapper.get() : nullptr; auto& smmu = system.Host1x().MemoryManager(); @@ -224,7 +252,10 @@ void nvhost_as_gpu::Nx2Sparse(u64 gpu_address, u64 size, bool create) { return; } nx2_sparse_ranges[gpu_address] = {size, device_address}; - gmmu->Map(gpu_address, device_address, size, Tegra::PTEKind::PITCH, false); + // On the GPU thread, as in Nx2Map. + system.GPU().RunInOrder([gmmu = gmmu.get(), gpu_address, device_address, size] { + gmmu->Map(gpu_address, device_address, size, Tegra::PTEKind::PITCH, false); + }); LOG_DEBUG(Service_NVDRV, "Switch 2 sparse region {:#x}+{:#x} -> device {:#x}", gpu_address, size, device_address); } diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h index 39b63ea369..37b6d0d4c4 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h @@ -238,6 +238,8 @@ private: struct Nx2DeviceRange { DAddr address; bool heap; ///< Mapped through the session's heap mapper (else an SMMU allocation) + bool shared{}; ///< A reference to the process's range in Nx2GpuMemoryRegistry + size_t asid{}; }; std::map<std::pair<u64, u64>, Nx2DeviceRange> nx2_device_ranges; void Nx2Release(u64 cpu_address, u64 size); diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp index 5998940afa..93d51cd538 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp @@ -113,12 +113,22 @@ NvResult nvhost_ctrl_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> case 0x6: return WrapFixedInlOut(this, &nvhost_ctrl_gpu::GetTPCMasks3, input, output, inline_output); - case 0x13: - LOG_DEBUG(Service_NVDRV, "(STUBBED) called."); - - // TODO (jarrodnorwell) - - return NvResult::NotImplemented; + case 0x13: { + // NVGPU_GPU_IOCTL_VSMS_MAPPING: an 8-byte argument struct plus a buffer of + // {gpc_index, tpc_index} byte pairs, one per virtual SM. The SDK treats a pair left at + // its 0xc0c0 sentinel as failure, so fill every pair: one GPC, two SMs per TPC. + if (command.raw != 0xc0084713 || input.size() != 8 || output.size() != 8 || + inline_output.size() < 2) { + return NvResult::BadParameter; + } + std::memcpy(output.data(), input.data(), output.size()); + for (size_t i = 0; i + 1 < inline_output.size(); i += 2) { + inline_output[i] = 0; // gpc_index + inline_output[i + 1] = static_cast<u8>(i / 4); // tpc_index + } + LOG_DEBUG(Service_NVDRV, "VSMS mapping for {} SMs", inline_output.size() / 2); + return NvResult::Success; + } default: break; } @@ -218,6 +228,11 @@ void ApplyNx2Characteristics(GC& gc) { if (remap) { gc.flags |= 1ULL << 35; } + // Engine support bits 40-44 (2D, 3D, compute, inline-to-memory, ZBC). The SDK filters its class + // enumeration by them, and NvRm's newer device path lists a class family in the device info + // only when its bit is set; without them Fire Emblem: Fortune's Weave aborts at boot and a + // device opened that way has no compute class (Fire Emblem's CUDA driver reports no device). + gc.flags |= 0x1FULL << 40; // The Switch 2 layout keeps nine s16 interface versions at 0x70..0x80 (like nvgpu's // *_ioctl_nr_last); its NvRm requires e.g. [0x78] >= 6 and enables features above 0xa, 0x1f diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp index 0cd72faed3..848c15d6f8 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp @@ -24,6 +24,7 @@ #include "video_core/engines/puller.h" #include "video_core/gpu.h" #include "video_core/host1x/host1x.h" +#include "video_core/memory_manager.h" namespace Service::Nvidia::Devices { namespace { @@ -435,6 +436,30 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL entries.submit_hashes.push_back(Tegra::HashPushbufferWords(words)); } } + // The present returns before the GPU thread reaches this frame's work (nx2_gsv.h, + // frames ahead), and Mario Kart World then refills command memory it already + // submitted: the end of a frame ran with the next frame's present image as its render + // target, leaving an image three frames old to be shown. Keep the words as submitted. + // EDEN_NX2_PB_SNAPSHOT=0: execute from memory. + static const bool snapshot = [] { + const char* value = std::getenv("EDEN_NX2_PB_SNAPSHOT"); + return value == nullptr || value[0] != '0'; + }(); + if (snapshot && channel_state->memory_manager) { + size_t words = 0; + for (const auto& entry : entries.command_lists) { + words += entry.size; + } + entries.submitted_words.resize(words); + entries.submitted_offsets.reserve(entries.command_lists.size()); + size_t offset = 0; + for (const auto& entry : entries.command_lists) { + entries.submitted_offsets.push_back(static_cast<u32>(offset)); + channel_state->memory_manager->ReadBlockUnsafe( + entry.addr, entries.submitted_words.data() + offset, entry.size * sizeof(u32)); + offset += entry.size; + } + } } auto& flags = params.flags; diff --git a/src/core/hle/service/nvdrv/devices/nx2_stub_device.h b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h index 869e17caf6..8dee9ccfb7 100644 --- a/src/core/hle/service/nvdrv/devices/nx2_stub_device.h +++ b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h @@ -5,6 +5,7 @@ #include <algorithm> #include <atomic> +#include <compare> #include <functional> #include <map> #include <mutex> @@ -166,12 +167,50 @@ public: { std::scoped_lock lk{device_mutex}; device_ranges.clear(); + shared_ranges.clear(); } std::scoped_lock lk{mutex}; mappings.clear(); sparse_regions.clear(); } + // Device (SMMU) range of a mapping outside the heap, shared by every address space of one + // process: the caches key memory by device address, so a range per address space made separate + // copies of the same memory, and presentation could read one the game never drew into. The + // first address space to map the range allocates it with `allocate` (0: out of space); every + // call takes a reference. + u64 AcquireDeviceRange(size_t asid, u64 cpu_address, u64 size, + const std::function<u64()>& allocate) { + std::scoped_lock lk{device_mutex}; + const SharedRangeKey key{asid, cpu_address, size}; + if (const auto it = shared_ranges.find(key); it != shared_ranges.end()) { + ++it->second.references; + return it->second.device_address; + } + const u64 device_address = allocate(); + if (device_address != 0) { + shared_ranges.emplace(key, SharedRange{device_address, 1}); + } + return device_address; + } + + // Drops a reference. True for the last one: the caller frees the range. + bool ReleaseDeviceRange(size_t asid, u64 cpu_address, u64 size) { + std::scoped_lock lk{device_mutex}; + const auto it = shared_ranges.find(SharedRangeKey{asid, cpu_address, size}); + if (it == shared_ranges.end() || --it->second.references != 0) { + return false; + } + shared_ranges.erase(it); + return true; + } + + // The process's session closed and its ASID can be reused: stop sharing its ranges. + void ForgetDeviceRanges(size_t asid) { + std::scoped_lock lk{device_mutex}; + std::erase_if(shared_ranges, [asid](const auto& entry) { return entry.first.asid == asid; }); + } + // Device (SMMU) address of a CPU range an address space mapped; used to present images. void RecordDeviceRange(u64 cpu_address, u64 size, u64 device_address) { std::scoped_lock lk{device_mutex}; @@ -197,6 +236,18 @@ private: std::mutex device_mutex; std::map<u64, std::pair<u64, u64>> device_ranges; // cpu start -> (size, device address) + struct SharedRangeKey { + size_t asid; + u64 cpu_address; + u64 size; + auto operator<=>(const SharedRangeKey&) const = default; + }; + struct SharedRange { + u64 device_address; + u32 references; + }; + std::map<SharedRangeKey, SharedRange> shared_ranges; + struct Mapping { u64 gpu_address; u64 cpu_address; diff --git a/src/core/hle/service/nvdrv/nvdrv.cpp b/src/core/hle/service/nvdrv/nvdrv.cpp index 12923496fb..7853d17235 100644 --- a/src/core/hle/service/nvdrv/nvdrv.cpp +++ b/src/core/hle/service/nvdrv/nvdrv.cpp @@ -133,6 +133,7 @@ Module::Module(Core::System& system) Devices::Nx2GpuMemoryRegistry::Get().Reset(); Tegra::Nx2ClearSemaphoreRegions(); Tegra::Nx2ClearSparseRegions(); + Tegra::Nx2ClearGpuWrittenPages(); Devices::g_nx2_driver = false; Tegra::g_nx2_ampere = false; Core::g_nx2_present_count = 0; diff --git a/src/core/hle/service/nvdrv/nvdrv_interface.cpp b/src/core/hle/service/nvdrv/nvdrv_interface.cpp index 020036d74b..2ee0d726a2 100644 --- a/src/core/hle/service/nvdrv/nvdrv_interface.cpp +++ b/src/core/hle/service/nvdrv/nvdrv_interface.cpp @@ -13,6 +13,7 @@ #include "core/hle/kernel/k_process.h" #include "core/hle/kernel/k_readable_event.h" #include "core/hle/service/ipc_helpers.h" +#include "core/hle/service/nvdrv/devices/nx2_stub_device.h" #include "core/hle/service/nvdrv/nvdata.h" #include "core/hle/service/nvdrv/nvdrv.h" #include "core/hle/service/nvdrv/nvdrv_interface.h" @@ -36,7 +37,10 @@ void NVDRV::Open(HLERequestContext& ctx) { const std::string device_name(Common::StringFromBuffer(buffer)); LOG_DEBUG(Service_NVDRV, "device_name={}", device_name); - if (device_name == "/dev/nvhost-prof-gpu") { + // Switch 2 lists nvhost-prof-gpu with the other GPU nodes and the CUDA driver (Fire + // Emblem's subsdk1) opens it, so only refuse it to Switch 1 callers. Switch 2 opens get + // the logged stub device. + if (device_name == "/dev/nvhost-prof-gpu" && !Devices::g_nx2_driver) { rb.Push<DeviceFD>(0); rb.PushEnum(NvResult::NotSupported); diff --git a/src/core/hle/service/nvnflinger/pixel_format.h b/src/core/hle/service/nvnflinger/pixel_format.h index b6e8b5f8a0..a38c8be777 100644 --- a/src/core/hle/service/nvnflinger/pixel_format.h +++ b/src/core/hle/service/nvnflinger/pixel_format.h @@ -19,6 +19,7 @@ enum class PixelFormat : u32 { Bgra8888 = 5, Rgba5551 = 6, Rgba4444 = 7, + Rgba1010102 = 43, }; } // namespace Service::android diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h index 628bdeda5b..37c797fa3f 100644 --- a/src/core/hle/service/nx2_gsv.h +++ b/src/core/hle/service/nx2_gsv.h @@ -21,6 +21,7 @@ #include <optional> #include <memory> #include <mutex> +#include <utility> #include <vector> #include "common/alignment.h" @@ -45,13 +46,17 @@ #include "core/hle/kernel/k_resource_limit.h" #include "core/hle/kernel/k_server_session.h" #include "core/hle/kernel/k_event.h" +#include "core/hle/kernel/k_hardware_timer.h" #include "core/hle/kernel/k_readable_event.h" #include "core/hle/kernel/k_session.h" +#include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/k_transfer_memory.h" +#include "core/hle/kernel/svc_results.h" #include "core/memory.h" #include "video_core/host1x/host1x.h" #include "video_core/host1x/nx2_video_decoder.h" #include "video_core/nx2.h" +#include "video_core/textures/decoders.h" #include "core/hle/service/hle_ipc.h" #include "core/hle/service/ipc_helpers.h" #include "core/hle/service/kernel_helpers.h" @@ -102,6 +107,14 @@ inline void TipcReplyWords(HLERequestContext& ctx, std::initializer_list<u32> wo TipcReplyWords(ctx, std::span<const u32>{words.begin(), words.size()}); } +// TIPC 0(u64 token) on the Switch 2 sessions: a handshake newer SDKs send first; the reply echoes +// the token. (The auto stub's reply only echoed it by leaving the request's words in place.) +inline void TipcHandshake(HLERequestContext& ctx, std::string_view session) { + const auto* w = ctx.CommandBuffer(); + LOG_INFO(Service, "{} TIPC 0 handshake {:#x}", session, u64{w[2]} | (u64{w[3]} << 32)); + TipcReplyWords(ctx, {w[2], w[3]}); +} + // Replies with success and a copied handle to a new session served by `iface`. template <class T> inline void ReturnRawSession(Core::System& system, HLERequestContext& ctx, std::shared_ptr<T> iface) { @@ -139,6 +152,8 @@ public: {103, &INx2SyncptSession::QueryCounter, "QueryCounter"}, {201, &INx2SyncptSession::RequestSlotEvent, "RequestSlotEvent (guessed)"}, {202, &INx2SyncptSession::CancelSlotEvent, "CancelSlotEvent (guessed)"}, + // Firmware commands without a known meaning: zeroed replies. + {21, &INx2SyncptSession::StubReplyZeroed, "Unknown21"}, }); } @@ -160,9 +175,7 @@ public: private: // Newer SDKs (title 1) open with TIPC 0(u64 token) and abort unless the reply echoes it. void Handshake(HLERequestContext& ctx) { - const auto* w = ctx.CommandBuffer(); - LOG_INFO(Service, "nx2:syncpt TIPC 0 handshake {:#x}", u64{w[2]} | (u64{w[3]} << 32)); - TipcReplyWords(ctx, {w[2], w[3]}); + TipcHandshake(ctx, "nx2:syncpt"); } void GetCount(HLERequestContext& ctx) { @@ -299,11 +312,14 @@ public: explicit INx2GpuSession(Core::System& system_) : ServiceFramework{system_, "nx2:gpu"}, m_context{system_, "nx2:gpu"} { RegisterHandlersTipc({ + {0, &INx2GpuSession::Handshake, "Handshake"}, {11, &INx2GpuSession::GetIdRange, "GetIdRange (guessed)"}, {12, &INx2GpuSession::GetIdEvent, "GetIdEvent (guessed)"}, {21, &INx2GpuSession::Start, "Start (guessed)"}, {22, &INx2GpuSession::OpenSubSession, "OpenSubSession (guessed)"}, {33, &INx2GpuSession::GetCount33, "Unknown33 (u32 count)"}, + {41, &INx2GpuSession::CreateGraphicsDevice, "CreateGraphicsDevice"}, + {42, &INx2GpuSession::DestroyGraphicsDevice, "DestroyGraphicsDevice"}, {201, &INx2GpuSession::GetUserPartition, "GetUserPartition (newer SDKs)"}, {202, &INx2GpuSession::GetGsvPartition, "GetGsvPartition (newer SDKs)"}, {203, &INx2GpuSession::GetVaRange, "GetVaRange (guessed)"}, @@ -316,10 +332,22 @@ public: {214, &INx2GpuSession::SubCreateGpuOnlyRegion, "CreateSparseRegion"}, {215, &INx2GpuSession::SubDestroySparseRegion, "DestroySparseRegion"}, {216, &INx2GpuSession::SubSparseMap, "ExecuteSparseMapRequests"}, + // Firmware commands without a known meaning: zeroed replies. + {23, &INx2GpuSession::StubReplyZeroed, "Unknown23"}, + {24, &INx2GpuSession::StubReplyZeroed, "Unknown24"}, + {25, &INx2GpuSession::StubReplyZeroed, "Unknown25"}, + {34, &INx2GpuSession::StubReplyZeroed, "Unknown34"}, + {51, &INx2GpuSession::StubReplyZeroed, "Unknown51"}, + {8001, &INx2GpuSession::StubReplyZeroed, "Unknown8001"}, + {8002, &INx2GpuSession::StubReplyZeroed, "Unknown8002"}, + {8003, &INx2GpuSession::StubReplyZeroed, "Unknown8003"}, + {8004, &INx2GpuSession::StubReplyZeroed, "Unknown8004"}, }); if (LazySparseRegions()) { Tegra::Nx2SetSparseBacking( - [this](u64 gpu_address, u64 size) { BackSparseWrite(gpu_address, size); }); + [this](u64 gpu_address, u64 size, const u8* data) { + BackSparseWrite(gpu_address, size, data); + }); } } @@ -334,6 +362,10 @@ public: } private: + void Handshake(HLERequestContext& ctx) { + TipcHandshake(ctx, "nx2:gpu"); + } + static constexpr u32 NumIds = 32; Kernel::KEvent* EventFor(u32 id) { @@ -395,9 +427,39 @@ private: static_cast<u32>(second), static_cast<u32>(second >> 32)}); } + // An empty partition is min = ~0, max = 0 (min > max). Fire Emblem: Fortune's Weave's SDK + // reads (0, 0) as a one-byte partition at address 0 and aborts in nn::gpu::sdk. void GetGsvPartition(HLERequestContext& ctx) { LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 202 [2]={:#x}, empty", ctx.CommandBuffer()[2]); - TipcReplyWords(ctx, {0, 0, 0, 0}); + TipcReplyWords(ctx, {~0U, ~0U, 0, 0}); + } + + // Fire Emblem: Fortune's Weave: 41(name u64, flags u64, debug label u64) returns a device id + // that later NvRm calls name; 0 is not a valid id. Only flags 0 has been seen. + void CreateGraphicsDevice(HLERequestContext& ctx) { + const auto* b = ctx.CommandBuffer(); + const u64 name = u64{b[2]} | (u64{b[3]} << 32); + const u64 flags = u64{b[4]} | (u64{b[5]} << 32); + u32 id = 0; + for (u32 candidate = 1; candidate < m_devices.size(); ++candidate) { + if (!m_devices[candidate]) { + m_devices[candidate] = true; + id = candidate; + break; + } + } + LOG_INFO(Service, "nx2:gpu 41 create graphics device name={:#x} flags={:#x} -> id {}", name, + flags, id); + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(id != 0 ? ResultSuccess : Kernel::ResultOutOfResource); + rb.Push(id); + } + + void DestroyGraphicsDevice(HLERequestContext& ctx) { + const u32 id = ctx.CommandBuffer()[2] & 0xff; + const bool existed = id < m_devices.size() && std::exchange(m_devices[id], false); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(existed ? ResultSuccess : Kernel::ResultNotFound); } void OpenSubSession(HLERequestContext& ctx) { @@ -551,12 +613,29 @@ private: // backed whole, they ran the pool dry and the later ones got no memory at all. // The region gets a device range of its own, unbacked pages read zero, and // BackSparseWrite gives a block memory before the GPU writes into it. - m_sparse_regions[start] = {size, std::vector<u8>(Common::DivCeil(size, SparseBlockSize))}; + m_sparse_regions[start] = { + .size = size, + .blocks = std::vector<u8>(Common::DivCeil(size, SparseBlockSize)), + }; Nvidia::Devices::Nx2GpuMemoryRegistry::Get().CreateSparse(start, size); Tegra::Nx2SparseRegionCreated(start, size, SparseBlockSize); Tegra::Nx2AddSparseSpan(start, size); LOG_INFO(Service, "nx2:gpu 214 sparse region [{:#x}, {:#x}] backed as the GPU writes it", start, last); + } else if (LazyGpuOnlyRegions()) { + // The rest is written by the host GPU, not through guest memory (title 1's and + // title 4's render targets, 285-300 MB regions): like a sparse region it gets a + // device range of its own and memory per block only when something writes guest + // memory there (downloads, copies). The texture cache sees ordinary memory. + m_sparse_regions[start] = { + .gpu_only = true, + .size = size, + .blocks = std::vector<u8>(Common::DivCeil(size, SparseBlockSize)), + }; + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().CreateSparse(start, size); + Tegra::Nx2AddSparseSpan(start, size); + LOG_INFO(Service, "nx2:gpu 214 GPU-only region [{:#x}, {:#x}] backed as it is written", + start, last); } else if (const auto address = BackGpuOnlyRegion(start, size)) { Nvidia::Devices::Nx2GpuMemoryRegistry::Get().MapAt(start, *address, size); LOG_INFO(Service, "nx2:gpu 214 GPU-only region [{:#x}, {:#x}] backed at {:#x} " @@ -665,6 +744,23 @@ private: TipcReplyWords(ctx, {static_cast<u32>(count), static_cast<u32>(u64{count} >> 32)}); } + static bool IsAllZero(const u8* data, size_t size) { + size_t i = 0; + for (; i + sizeof(u64) <= size; i += sizeof(u64)) { + u64 word; + std::memcpy(&word, data + i, sizeof(word)); + if (word != 0) { + return false; + } + } + for (; i < size; ++i) { + if (data[i] != 0) { + return false; + } + } + return true; + } + // Returns the pages of destroyed regions the GPU thread is done with to the kernel. void FreeReleasedRegions() { auto* const process = system.ApplicationProcess(); @@ -772,7 +868,7 @@ private: // usually on the GPU thread): the blocks the write touches get memory, cleared, behind the // region's device pages. The GPU mapping stays as it is, so the images over the region keep // their storage and only see the write. - void BackSparseWrite(u64 gpu_address, u64 size) { + void BackSparseWrite(u64 gpu_address, u64 size, const u8* data) { struct Pending { u64 region; u64 block; @@ -795,6 +891,7 @@ private: } }; bool waits_for_other = false; + u64 zero_bytes = 0; { std::unique_lock lk{m_region_mutex}; for_regions([&](u64 start, SparseRegion& region, u64 first, u64 last) { @@ -805,6 +902,18 @@ private: region.blocks[block] = BlockWritten; Tegra::Nx2SparseBlockWritten(start, block); } else if (region.blocks[block] == BlockEmpty) { + // Zeros over a block nobody wrote change nothing: it reads zero without + // memory. Title 1 clears ~1 GB of them at boot. + const u64 block_begin = start + block * SparseBlockSize; + const u64 write_begin = (std::max)(gpu_address, block_begin); + const u64 write_end = (std::min)( + {end, block_begin + SparseBlockSize, start + region.size}); + if (data != nullptr && + IsAllZero(data + (write_begin - gpu_address), write_end - write_begin) && + !Tegra::Nx2SparseBlockIsWritten(start, block)) { + zero_bytes += write_end - write_begin; + continue; + } region.blocks[block] = BlockPending; todo.push_back({start, block}); } else if (region.blocks[block] == BlockPending) { @@ -813,6 +922,16 @@ private: } }); } + if (zero_bytes != 0) { + std::scoped_lock lk{m_region_mutex}; + const u64 before = m_sparse_zero_bytes; + m_sparse_zero_bytes += zero_bytes; + if (before == 0 || (before >> 28) != (m_sparse_zero_bytes >> 28)) { + LOG_INFO(Service, "nx2:gpu {} MiB of zeros over unwritten sparse blocks left out " + "(they read zero without memory)", + m_sparse_zero_bytes >> 20); + } + } // Allocating and mapping happens outside the lock (see AllocateGpuOnlyBacking). for (const auto& [start, block] : todo) { u64 block_size = SparseBlockSize; @@ -896,6 +1015,15 @@ private: return lazy; } + // EDEN_NX2_GPU_ONLY_LAZY=0 backs GPU-only regions whole, as before. + static bool LazyGpuOnlyRegions() { + static const bool lazy = [] { + const char* value = std::getenv("EDEN_NX2_GPU_ONLY_LAZY"); + return value == nullptr || value[0] != '0'; + }(); + return lazy; + } + static constexpr u32 SparseRegionFlag = 0x200; ///< Word 7 of 214 (0x60c arrays, 0xa0c pools) static constexpr u64 SparseBlockSize = 4ULL << 20; static constexpr u8 BlockEmpty = 0; @@ -904,6 +1032,7 @@ private: static constexpr u8 BlockPending = 3; ///< Being allocated and mapped by BackSparseWrite static constexpr u8 BlockWritten = 4; ///< Written in a region the game maps tiles into struct SparseRegion { + bool gpu_only{}; ///< A GPU-only region (no sparse flag), backed lazily as well u64 size; std::vector<u8> blocks; ///< BlockEmpty / Backed / Failed / Pending / Written per block bool warned{}; ///< Logged that the pool ran out for its blocks @@ -913,6 +1042,7 @@ private: std::map<u64, SparseRegion> m_sparse_regions; ///< By GPU address u64 m_sparse_blocks{}; ///< Sparse blocks backed so far u64 m_sparse_backed_bytes{}; ///< Sparse region memory given out so far + u64 m_sparse_zero_bytes{}; ///< Zeros written over unwritten sparse blocks, left out u64 m_sparse_released_bytes{}; ///< Blocks given back once the game mapped tiles there // nn::gpu::sdk::SparseMapRequest (216), filled from NvRm's nvgpu_as_remap_op. @@ -1246,6 +1376,47 @@ private: std::array<Kernel::KEvent*, NumIds> m_events{}; std::map<u32, GpuMemory> m_memory; u32 m_next_memory_handle{1}; + std::array<bool, 64> m_devices{}; // gpu 41 graphics device ids (0 unused) + + // Every 30 s: the backing of sparse blocks and GPU-only regions is marked cold. The game + // writes it once (texture uploads by 2D blits: title 1 fills 2-4 GB of texture pools) and it + // is read again only when the texture cache builds an image over it, so the OS may compress or + // page it out before anything else. Last member: stops before the regions are destroyed. + void HintColdBacking() { + struct Range { + u64 physical_offset; + u64 virtual_address; + u64 size; + }; + std::vector<Range> ranges; + { + std::scoped_lock lk{m_region_mutex}; + for (const auto& [gpu_address, region] : m_gpu_only_regions) { + u64 virtual_address = region.cpu_address; + for (const auto& block : *region.pages) { + ranges.push_back({GetInteger(block.GetAddress()) - Core::DramMemoryMap::Base, + virtual_address, block.GetSize()}); + virtual_address += block.GetSize(); + } + } + } + auto& buffer = system.DeviceMemory().buffer; + for (const Range& range : ranges) { + buffer.HintCold(range.physical_offset, range.size, range.virtual_address); + } + } + std::jthread m_cold_hint_thread{[this](std::stop_token stop) { + Common::SetCurrentThreadName("nx2:ColdHint"); + Common::SetCurrentThreadPriority(Common::ThreadPriority::Low); + while (!stop.stop_requested()) { + for (int i = 0; i < 30 && !stop.stop_requested(); ++i) { + std::this_thread::sleep_for(std::chrono::seconds{1}); + } + if (!stop.stop_requested()) { + HintColdBacking(); + } + } + }}; }; // Native memory and images handed out by the Switch 2 graphics server (gsv 111/114/423/429/315), @@ -1264,6 +1435,10 @@ struct Nx2NativeObjects { u64 offset{}; // into its memory (images of a decoder pool share one) bool window_pool{}; bool decoder{}; // a video decoder output image (gsv 911) + // A decoder image the client reads on the GPU (gsv 428/426): written block-linear. + bool gpu_read{}; + u32 gpu_maps{}; // outstanding 426 maps + bool mapped_here{}; // 426 added its GPU mapping (no other mapping covered it) }; std::mutex mutex; std::map<u32, Memory> memories; @@ -1347,6 +1522,34 @@ private: std::map<u32, Kernel::KEvent*> m_events; }; +// nn::camera::sdk camera context (gsv 602). GameChat builds its local camera video storage from it +// and polls it every frame. Nothing about the commands is known beyond the arguments seen (3 and +// 21 take the camera name "Default", 29 a u32 0x180 = 384, the width of the 384x216 camera images), +// and the game runs on an empty success, so they answer that without a log line per call. +class INx2CameraSession final : public ServiceFramework<INx2CameraSession> { +public: + explicit INx2CameraSession(Core::System& system_) : ServiceFramework{system_, "nx2:camera"} { + RegisterHandlersTipc({ + {3, &INx2CameraSession::Stub, "Unknown3 (camera name \"Default\")"}, + {21, &INx2CameraSession::Stub, "Unknown21 (camera name \"Default\")"}, + {29, &INx2CameraSession::Stub, "Unknown29 (u32 0x180, polled every frame)"}, + }); + } + +private: + void Stub(HLERequestContext& ctx) { + const u32 command = ctx.GetCommand(); + if (m_logged.insert(command).second) { + LOG_WARNING(Service, "(STUBBED) nx2:camera TIPC {} [2]={:#x} [3]={:#x}, returning success", + command, ctx.CommandBuffer()[2], ctx.CommandBuffer()[3]); + } + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + std::set<u32> m_logged; +}; + // Video decoder output images (nn::codec image pools, gsv 991/911/961). The preference // (MakeDecodeImagePreference) holds an output layout code at word 1 and (width - 1) | // (height - 1) << 16 at word 2. Codes come in pairs, one per CodecMode: 1/4 NV12 (format 3, what @@ -1401,19 +1604,73 @@ public: : ServiceFramework{system_, "nx2:vi2"}, m_context{system_, "nx2:vi2"}, m_objects{std::move(objects_)} { RegisterHandlersTipc({ + {0, &INx2Vi2Session::Handshake, "Handshake"}, {1, &INx2Vi2Session::ReturnOne, "Unknown1"}, {3, &INx2Vi2Session::ReturnOne, "GetDisplayCount (guessed)"}, {5, &INx2Vi2Session::GetTimestampRecorderCount, "GetTimestampRecorderCount (guessed)"}, {11, &INx2Vi2Session::ReturnEvent, "Unknown11"}, - {31, &INx2Vi2Session::GetVsyncEvent, "GetVsyncEvent (guessed)"}, - {32, &INx2Vi2Session::GetVsyncEvent, "GetDisplayVsyncEvent (guessed)"}, - // Title 1 calls these ~75 times a second with the display name "default"; the - // auto stub's empty success reply works, but logged two lines per call (a 50 MB log - // in an 18 minute session). - {104, &INx2Vi2Session::SucceedQuietly, "Unknown104 (display \"default\")"}, - {105, &INx2Vi2Session::SucceedQuietly, "Unknown105 (display \"default\")"}, + {31, &INx2Vi2Session::AcquireNativeEvent, "AcquireNativeEvent"}, + {32, &INx2Vi2Session::DuplicateNativeEvent, "DuplicateNativeEvent"}, + {261, &INx2Vi2Session::BindVsync, "BindDisplayVsyncEvent"}, + {262, &INx2Vi2Session::UnbindNativeEvent, "UnbindDisplayVsyncEvent"}, + {263, &INx2Vi2Session::BindChange, "BindDisplayRefreshRateEvent"}, + {264, &INx2Vi2Session::UnbindNativeEvent, "UnbindDisplayRefreshRateEvent"}, + {265, &INx2Vi2Session::BindChange, "BindDisplayHdrStatusEvent"}, + {266, &INx2Vi2Session::UnbindNativeEvent, "UnbindDisplayHdrStatusEvent"}, + {267, &INx2Vi2Session::BindChange, "BindDisplayHdrPaperWhiteEvent"}, + {268, &INx2Vi2Session::UnbindNativeEvent, "UnbindDisplayHdrPaperWhiteEvent"}, + {269, &INx2Vi2Session::BindChange, "BindDisplayBufferMaxResolutionEvent"}, + {270, &INx2Vi2Session::UnbindNativeEvent, "UnbindDisplayBufferMaxResolutionEvent"}, + {271, &INx2Vi2Session::BindChange, "BindDisplayModeCapabilityEvent"}, + {272, &INx2Vi2Session::UnbindNativeEvent, "UnbindDisplayModeCapabilityEvent"}, + // Title 1 calls these ~75 times a second with the display name "default"; quiet + // handlers (the auto stub logged two lines per call: a 50 MB log in an 18 minute + // session). + {103, &INx2Vi2Session::QueryDisplayRefreshRate, "QueryDisplayRefreshRate"}, + {104, &INx2Vi2Session::QueryDisplayHdrStatus, "QueryDisplayHdrStatus"}, + {105, &INx2Vi2Session::QueryDisplayHdrPaperWhite, "QueryDisplayHdrPaperWhite"}, + {212, &INx2Vi2Session::GetLayerBufferBits, "GetLayerBufferBits"}, + {106, &INx2Vi2Session::GetBufferMaxResolution, "GetBufferMaxResolution"}, {228, &INx2Vi2Session::CancelClientBuffer, "CancelClientBuffer"}, {229, &INx2Vi2Session::PresentClientBuffer, "PresentClientBuffer"}, + // Firmware commands without a known meaning: zeroed replies. + {13, &INx2Vi2Session::StubReplyZeroed, "Unknown13"}, + {51, &INx2Vi2Session::StubReplyZeroed, "Unknown51"}, + {101, &INx2Vi2Session::StubReplyZeroed, "Unknown101"}, + {102, &INx2Vi2Session::StubReplyZeroed, "Unknown102"}, + {107, &INx2Vi2Session::StubReplyZeroed, "Unknown107"}, + {108, &INx2Vi2Session::StubReplyZeroed, "Unknown108"}, + {111, &INx2Vi2Session::StubReplyZeroed, "Unknown111"}, + {112, &INx2Vi2Session::StubReplyZeroed, "Unknown112"}, + {113, &INx2Vi2Session::StubReplyZeroed, "Unknown113"}, + {202, &INx2Vi2Session::StubReplyZeroed, "Unknown202"}, + {230, &INx2Vi2Session::StubReplyZeroed, "Unknown230"}, + {231, &INx2Vi2Session::StubReplyZeroed, "Unknown231"}, + {232, &INx2Vi2Session::StubReplyZeroed, "Unknown232"}, + {251, &INx2Vi2Session::StubReplyZeroed, "Unknown251"}, + {252, &INx2Vi2Session::StubReplyZeroed, "Unknown252"}, + {261, &INx2Vi2Session::StubReplyZeroed, "Unknown261"}, + {262, &INx2Vi2Session::StubReplyZeroed, "Unknown262"}, + {263, &INx2Vi2Session::StubReplyZeroed, "Unknown263"}, + {264, &INx2Vi2Session::StubReplyZeroed, "Unknown264"}, + {265, &INx2Vi2Session::StubReplyZeroed, "Unknown265"}, + {266, &INx2Vi2Session::StubReplyZeroed, "Unknown266"}, + {267, &INx2Vi2Session::StubReplyZeroed, "Unknown267"}, + {268, &INx2Vi2Session::StubReplyZeroed, "Unknown268"}, + {269, &INx2Vi2Session::StubReplyZeroed, "Unknown269"}, + {270, &INx2Vi2Session::StubReplyZeroed, "Unknown270"}, + {271, &INx2Vi2Session::StubReplyZeroed, "Unknown271"}, + {272, &INx2Vi2Session::StubReplyZeroed, "Unknown272"}, + {301, &INx2Vi2Session::StubReplyZeroed, "Unknown301"}, + {302, &INx2Vi2Session::StubReplyZeroed, "Unknown302"}, + {351, &INx2Vi2Session::StubReplyZeroed, "Unknown351"}, + {352, &INx2Vi2Session::StubReplyZeroed, "Unknown352"}, + {353, &INx2Vi2Session::StubReplyZeroed, "Unknown353"}, + {354, &INx2Vi2Session::StubReplyZeroed, "Unknown354"}, + {355, &INx2Vi2Session::StubReplyZeroed, "Unknown355"}, + {356, &INx2Vi2Session::StubReplyZeroed, "Unknown356"}, + {358, &INx2Vi2Session::StubReplyZeroed, "Unknown358"}, + {359, &INx2Vi2Session::StubReplyZeroed, "Unknown359"}, }); } @@ -1433,6 +1690,10 @@ public: } private: + void Handshake(HLERequestContext& ctx) { + TipcHandshake(ctx, "nx2:vi2"); + } + // 229(u64 client token, u32 slot, u32 image, ...fences): result, u64 mask of buffer slots the // client may acquire next, four zero words. Nothing is shown yet; every slot is handed back // at once (changes.patch's default mask is 0x7fff). @@ -1457,16 +1718,28 @@ private: LOG_DEBUG(Service, "nx2:vi2 present #{} fence ({}, {:#x}), waiting for ({}, {:#x}); now {:#x}", m_presents, ~w[22], w[24], m_previous_fence_id, m_previous_fence_value, ~w[22] < 192 ? system.Host1x().GetSyncpointManager().GetGuestSyncpointValue(~w[22]) : 0U); + // Only run a frame ahead while the GPU keeps up. With the previous frame still on the + // GPU here, the GPU is the limit and running ahead gains nothing; and Splatoon Raiders, + // which reuses its command memory as if the GPU were at most a frame behind, rewrote + // command lists Eden had not read yet (misread commands, then crashes, with a slow + // GPU). Wait for this frame too then. + const bool gpu_behind = + m_previous_fence_id < 192 && + !system.Host1x().GetSyncpointManager().IsReadyGuest(m_previous_fence_id, + m_previous_fence_value); WaitForFrame(m_previous_fence_id, m_previous_fence_value); + if (gpu_behind) { + WaitForFrame(~w[22], w[24]); + } m_previous_fence_id = ~w[22]; m_previous_fence_value = w[24]; } else { WaitForFrame(~w[22], w[24]); } const auto waited = std::chrono::steady_clock::now(); - Composite(w[5], ~w[22], w[24]); + Composite(w[5], ~w[22], w[24], w[7]); const auto composited = std::chrono::steady_clock::now(); - PaceToVblank(); + PaceToVblank(w[12]); const auto paced = std::chrono::steady_clock::now(); // Where each frame's time goes on the present thread, averaged per 600 presents: the // game is blocked here while Eden's GPU thread catches up (fence) and composites. @@ -1522,18 +1795,39 @@ private: // present faster than that. Every slot is reported free here: hold each present until the // next vblank instead (title 4 ran at up to 160 fps, its timing and audio with it). std::chrono::steady_clock::time_point m_next_vblank{}; + u32 m_swap_word{}; - void PaceToVblank() { + // timing: present word 12, (swap interval << 16) | refresh rate. Fire Emblem: Fortune's Weave + // sends 0x1003c during its boot logos and 0x2003c (two vblanks per frame, 30 fps) after; paced + // at every vblank it ran at double speed. EDEN_NX2_SWAP_INTERVAL=<n> forces the interval. + void PaceToVblank(u32 timing) { if (!Settings::values.use_speed_limit.GetValue() || Settings::SpeedLimit() == 0) { return; } - const auto interval = std::chrono::nanoseconds{100'000'000'000LL / 60 / Settings::SpeedLimit()}; + static const u32 forced_interval = [] { + const char* value = std::getenv("EDEN_NX2_SWAP_INTERVAL"); + return value ? static_cast<u32>(std::strtoul(value, nullptr, 10)) : 0U; + }(); + const u32 swap_interval = + forced_interval != 0 ? forced_interval : std::clamp<u32>(timing >> 16, 1, 4); + const u32 refresh_rate = (timing & 0xffff) >= 30 && (timing & 0xffff) <= 240 ? timing & 0xffff : 60; + if (timing != std::exchange(m_swap_word, timing)) { + LOG_INFO(Service, "nx2:vi2 present #{} swap interval {} at {} Hz (word 12 = {:#x})", + m_presents, swap_interval, refresh_rate, timing); + } + const auto interval = std::chrono::nanoseconds{ + 100'000'000'000LL * swap_interval / refresh_rate / Settings::SpeedLimit()}; const auto now = std::chrono::steady_clock::now(); if (now > m_next_vblank + interval) { - // Running slower than 60 fps: show at once and don't bank the lost time. + // Running slower than the swap interval: show at once and don't bank the lost time. m_next_vblank = now; } else if (now < m_next_vblank) { - std::this_thread::sleep_until(m_next_vblank); + // Sleep this service thread in the guest kernel. It runs on guest core 3, and a host + // sleep held that core for the whole wait: game threads scheduled there ran only + // between presents (Fortune's Weave's loader read one RomFS block per frame). + const auto wait = std::chrono::duration_cast<std::chrono::nanoseconds>(m_next_vblank - now); + static_cast<void>(Kernel::GetCurrentThread(kernel).Sleep( + kernel, kernel.HardwareTimer().GetTick() + wait.count() + 2)); } m_next_vblank += interval; } @@ -1586,9 +1880,12 @@ private: // Shows a presented native image: word 5 is the image id, ~word 22 / word 24 the GPU fence // (syncpoint, value) to wait for (layout from changes.patch). The image is a block-linear // RGBA8 surface in the console's native GOB layout at GPU address == CPU address. - void Composite(u32 image_id, u32 fence_id, u32 fence_value) { + // shown_size: the present's word 7, ((height - 1) << 16) | (width - 1) of the area the game drew + // into from the image's top-left corner (0 when it gives none). + void Composite(u32 image_id, u32 fence_id, u32 fence_value, u32 shown_size) { u64 address{}; Nx2ImageLayout layout{}; + u32 image_format{}; { std::scoped_lock lk{m_objects->mutex}; const auto image = m_objects->images.find(image_id); @@ -1601,6 +1898,7 @@ private: } address = memory->second.address; layout = Nx2CalculateImageLayout(image->second.preference.data()); + image_format = image->second.preference[6]; } const auto device = Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address); if (!device) { @@ -1610,6 +1908,16 @@ private: } u32 crop_width = layout.width; u32 crop_height = layout.height; + // The present says how much of the image the game drew: Kirby Air Riders draws 1920x1080 + // into 2560x1440 images, Mario Kart World 2560x1440 into 3840x2160 ones. Showing the whole + // image put the frame in the top-left corner. + const u32 shown_width = (shown_size & 0xFFFF) + 1, shown_height = (shown_size >> 16) + 1; + const bool has_shown_size = shown_size != 0 && shown_width <= layout.width && + shown_height <= layout.height; + if (has_shown_size) { + crop_width = shown_width; + crop_height = shown_height; + } // Handheld titles keep their docked present images (up to 3840x2160) and draw the frame // into the top-left 1920x1080, the handheld panel's resolution; the console's display // scans out only that region. Showing whole images put the frame in a corner inside black @@ -1617,7 +1925,8 @@ private: if (!Settings::IsDockedMode()) { crop_width = (std::min)(crop_width, 1920U); crop_height = (std::min)(crop_height, 1080U); - } else if (system.GetApplicationProcessProgramID() == 0x0400C3F00006E000ULL && + } else if (!has_shown_size && + system.GetApplicationProcessProgramID() == 0x0400C3F00006E000ULL && layout.width == 3840 && layout.height == 2160) { // Mario Kart World docked draws 2560x1440 into the top-left of its 4K images. crop_width = 2560; @@ -1639,8 +1948,9 @@ private: (u64{crop_width} << 16) | crop_height; if (shown != m_shown_layout) { m_shown_layout = shown; - LOG_INFO(Service, "nx2:vi2 composite image {} layout {}x{}, crop {}x{}", - image_id, layout.width, layout.height, crop_width, crop_height); + LOG_INFO(Service, + "nx2:vi2 composite image {} layout {}x{}, drawn area {:#x}, crop {}x{}", + image_id, layout.width, layout.height, shown_size, crop_width, crop_height); } std::vector<Tegra::FramebufferConfig> layers{Tegra::FramebufferConfig{ .address = *device, @@ -1648,7 +1958,9 @@ private: .width = layout.width, .height = layout.height, .stride = static_cast<u32>(layout.pitch / 4), - .pixel_format = android::PixelFormat::Rgba8888, + // Preference word 6: 1 = RGBA8, 4 = RGB10A2. + .pixel_format = image_format == 4 ? android::PixelFormat::Rgba1010102 + : android::PixelFormat::Rgba8888, .transform_flags = android::BufferTransformFlags::Unset, .crop_rect = {0, 0, static_cast<int>(crop_width), static_cast<int>(crop_height)}, .blending = Tegra::BlendMode::Opaque, @@ -1747,15 +2059,66 @@ private: static_cast<u32>(AvailableSlots >> 32), 0, 0, 0, 0}); } + // nn::vi2::QueryDisplayRefreshRate(display name): u32 refresh rate in Hz, then a bool (a u64 + // in all). nn::vi::detail::DefaultDisplay::QueryRefreshRate takes only -1, 60 or 120 and fails + // on anything else; Minecraft polls it and never draws while it fails. A one-word reply left + // the bool to whatever the message buffer held. + void QueryDisplayRefreshRate(HLERequestContext& ctx) { + TipcReplyWords(ctx, {60, 0}); + } + + // nn::vi2::QueryDisplayHdrPaperWhite(display name): u16 paper white in nits. The empty reply + // left it to the message buffer's leftovers; answer the BT.2408 reference white. + void QueryDisplayHdrPaperWhite(HLERequestContext& ctx) { + TipcReplyWords(ctx, {203}); + } + + // nn::vi2::QueryDisplayHdrStatus(display name): u64 HdrOutputFormat, u16. The sdk's + // DefaultDisplay::QueryHdrStatus takes format 1 as HDR output and fails with 0x272 on + // anything above 1. With the empty auto stub reply it read the request's leftover words as + // the format; Kirby Air Riders re-queries whenever its HDR status event fires and rebuilt its + // window images every few frames, which showed black. Report SDR output. + // nn::vi2::QueryDisplayHdrStatus(display name): u64 output format, u16 peak luminance in + // nits. The sdk takes format 1 as SDR and 2 as HDR and fails the query on any other value. + // Report SDR output with a typical SDR panel's peak. + void QueryDisplayHdrStatus(HLERequestContext& ctx) { + TipcReplyWords(ctx, {1, 0, 400}); + } + + // 212(u64 layer): u64 bits of the layer's buffer slots that are ready. The client polls it once + // a second until some are set; it only got going because the auto stub's reply happened to + // hold non-zero leftovers. Report the same slots 228/229 hand back. + void GetLayerBufferBits(HLERequestContext& ctx) { + constexpr u64 AvailableSlots = 0x7fff; + TipcReplyWords(ctx, {static_cast<u32>(AvailableSlots), static_cast<u32>(AvailableSlots >> 32)}); + } + void ReturnOne(HLERequestContext& ctx) { LOG_WARNING(Service, "(STUBBED) nx2:vi2 TIPC {}, returning 1", ctx.GetCommand()); TipcReplyWords(ctx, {1}); } - // The same reply as the auto stub (success, no data), without its log lines. - void SucceedQuietly(HLERequestContext& ctx) { - IPC::ResponseBuilder rb{ctx, 2}; - rb.Push(ResultSuccess); + // The highest buffer resolution mode of the "default" display. Fire Emblem: Fortune's Weave picks + // its output from it: 1 -> 1280x720, 2 -> 1920x1080, 4 and 8 -> 2560x1440; 0 (the auto stub's + // reply), 3 and 5 fall back to 720p. Docked reports 4, handheld 2 (the screen is 1080p); + // EDEN_VI2_MAX_RESOLUTION_MODE=<n> overrides. + void GetBufferMaxResolution(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const u64 name = u64{packet[2]} | (u64{packet[3]} << 32); + constexpr u64 DefaultDisplay = 0x00746c7561666564ULL; // "default" + Result result = Kernel::ResultInvalidCombination; + u64 mode{}; + if (packet[0] == 0x7a && packet[1] == 2) { + result = Result{ErrorModule::VI, 7}; + if (name == DefaultDisplay) { + static const char* forced = std::getenv("EDEN_VI2_MAX_RESOLUTION_MODE"); + mode = forced ? std::strtoull(forced, nullptr, 0) : Settings::IsDockedMode() ? 4 : 2; + result = ResultSuccess; + } + } + IPC::ResponseBuilder rb{ctx, 4}; + rb.Push(result.raw); + rb.Push(mode); } // Must be < 8; nn::vi::detail::LayerHistory creates 7 timestamp recorders per layer. @@ -1774,17 +2137,105 @@ private: rb.PushCopyObjects(ctx, event->GetReadableEvent()); } - // 31(u32 kind) and 32(u32 display): events signalled on every vblank. Title 1's main thread - // waits on the 32 event until it has fired twice before it submits any GPU work, and hung - // there while it never fired. - void GetVsyncEvent(HLERequestContext& ctx) { - LOG_INFO(Service, "nx2:vi2 {} vsync event, [2]={:#x}", ctx.GetCommand(), ctx.CommandBuffer()[2]); - auto* event = m_context.CreateEvent(fmt::format("nx2:vi2:Vsync{}", m_events.size())); - m_events.push_back(event); - { - std::scoped_lock lk{m_vsync->mutex}; - m_vsync->events.push_back(event); + // 31(u32 index < 8): the session's event for that slot (the sdk picks a free index). The + // 261..272 commands bind a slot to a display event source and unbind it again. Until a slot + // is bound it fires on every vblank, as all of them used to. + void AcquireNativeEvent(HLERequestContext& ctx) { + const u32 index = ctx.CommandBuffer()[2]; + if (index >= m_native_events.size()) { + LOG_ERROR(Service, "nx2:vi2 31 index {} out of range", index); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultUnknown); + return; + } + LOG_INFO(Service, "nx2:vi2 31 native event {}", index); + ReplyNativeEvent(ctx, index); + } + + // 32(u32 index, copy handle): the client hands back its handle for an acquired slot and gets + // the slot's event again; the handle descriptor and handle come before the index. Games call + // it for slot 0 right after binding it to vsync. Title 1's main thread waits on this event + // until it has fired twice before it submits any GPU work, and hung there while it never + // fired. + void DuplicateNativeEvent(HLERequestContext& ctx) { + const u32* words = ctx.CommandBuffer(); + u32 data = 2; + if (words[1] >> 31) { + const u32 descriptor = words[2]; + data = 3 + ((descriptor >> 1) & 0xf) + ((descriptor >> 5) & 0xf) + (descriptor & 1 ? 2 : 0); + } + const u32 index = words[data]; + if (index >= m_native_events.size()) { + LOG_ERROR(Service, "nx2:vi2 32 index {} out of range", index); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultUnknown); + return; } + LOG_INFO(Service, "nx2:vi2 32 native event {}", index); + ReplyNativeEvent(ctx, index); + } + + // 261(u64 display name, u32 index): fire the slot on every vblank. + void BindVsync(HLERequestContext& ctx) { + const u32 index = ctx.CommandBuffer()[4]; + LOG_INFO(Service, "nx2:vi2 261 native event {} -> vsync", index); + if (index < m_native_events.size() && m_native_events[index]) { + SetVsyncSlot(index, true); + } + TipcReplyWords(ctx, {}); + } + + // 263/265/267/269/271(u64 display name, u32 index): fire the slot when the display's refresh + // rate, HDR status, HDR paper white, buffer max resolution or mode capability changes, so the + // game re-queries it. None of them change here: fire once so the game reads the current + // state, then leave the slot quiet. Kirby Air Riders re-queried its HDR status on every + // present while this slot fired with vsync. + void BindChange(HLERequestContext& ctx) { + const u32 index = ctx.CommandBuffer()[4]; + LOG_INFO(Service, "nx2:vi2 {} native event {} -> change notification", ctx.GetCommand(), index); + if (index < m_native_events.size() && m_native_events[index]) { + SetVsyncSlot(index, false); + m_native_events[index]->Signal(system.Kernel()); + } + TipcReplyWords(ctx, {}); + } + + // 262..272 even (u64 display name, u32 index): detach the slot from its source. + void UnbindNativeEvent(HLERequestContext& ctx) { + const u32 index = ctx.CommandBuffer()[4]; + LOG_INFO(Service, "nx2:vi2 {} native event {} unbound", ctx.GetCommand(), index); + if (index < m_native_events.size() && m_native_events[index]) { + SetVsyncSlot(index, false); + } + TipcReplyWords(ctx, {}); + } + + void ReplyNativeEvent(HLERequestContext& ctx, u32 index) { + auto*& event = m_native_events[index]; + if (!event) { + event = m_context.CreateEvent(fmt::format("nx2:vi2:Native{}", index)); + m_events.push_back(event); + SetVsyncSlot(index, true); + StartVsyncTimer(); + } + IPC::ResponseBuilder rb{ctx, 2, 1, 0}; + rb.Push(ResultSuccess); + rb.PushCopyObjects(ctx, event->GetReadableEvent()); + } + + void SetVsyncSlot(u32 index, bool on) { + auto* event = m_native_events[index]; + std::scoped_lock lk{m_vsync->mutex}; + auto& list = m_vsync->events; + const auto it = std::find(list.begin(), list.end(), event); + if (on && it == list.end()) { + list.push_back(event); + } else if (!on && it != list.end()) { + list.erase(it); + } + } + + void StartVsyncTimer() { if (!m_vsync_timer) { m_vsync_timer = Core::Timing::CreateEvent( "nx2:vi2 vsync", @@ -1805,14 +2256,12 @@ private: }()}; system.CoreTiming().ScheduleLoopingEvent(VblankInterval, VblankInterval, m_vsync_timer); } - IPC::ResponseBuilder rb{ctx, 2, 1, 0}; - rb.Push(ResultSuccess); - rb.PushCopyObjects(ctx, event->GetReadableEvent()); } KernelHelpers::ServiceContext m_context; std::shared_ptr<Nx2NativeObjects> m_objects; std::vector<Kernel::KEvent*> m_events; + std::array<Kernel::KEvent*, 8> m_native_events{}; ///< 31's slots, also listed in m_events // Shared with the vsync timer callback, which can outlive the session by one call. struct VsyncState { std::mutex mutex; @@ -1837,7 +2286,9 @@ inline bool Nx2FirstVideoCommand(std::string_view service, u32 command) { // The Switch 2 video decoder (nn::codec::video::detail::ipc). gsv 902/906/1002 hand out a session // whose TIPC 1 (CreateVideoDecoderService: handle at word 3, result at word 4) creates a decoder // service, another session of this class. Layouts read from the SDK's nn::codec VP9 decoder: -// - 200(copy handle: work buffer transfer memory, u64 size, u32, u32, u32): initialize. +// - 200(copy handle: work buffer transfer memory, u64 size, u32, u32, u32): initialize. 201 adds a +// fourth u32 (a flag that 200 leaves at 1) and 202 that and a u64: the same initialization. +// 210(copy handle, u32 x4) replies only a result. // - 101(A: codec name, "video/vp9\0"; u32 width, u32 height, u32 rate, u32): configure. // - 3(): the submission-ready event (copy handle at word 3, result at word 4). // - 6 / 207 / 8: start / stop / flush; 115: finalize. @@ -1848,6 +2299,8 @@ inline bool Nx2FirstVideoCommand(std::string_view service, u32 command) { // rewritten once that handle (the client's reads) has passed. Reply the image's next handle. // - 212(u32 image id): unregister it. // - 109(A: one compressed frame, u64 pts in us, u8 end of stream): submit input; reply a u64. +// 400 is the same with a second A buffer after the frame, a 0x838-byte block of frame +// information; the decoder here only needs the frame. // - 213(): the next finished frame (16 words): u32 image id, u32 width, u32 height, u32 width, // u32 height, u32 colour space, TimeSpan pts (ns), u8 end of stream. // A NativeSyncHandle is {u32 SyncCounterId (~syncpoint), u32 mode, u64 value}; clients wait on it @@ -1869,11 +2322,15 @@ public: {109, &INx2VideoSession::SubmitInput, "SubmitInputBuffer"}, {115, &INx2VideoSession::Succeed, "Finalize"}, {200, &INx2VideoSession::Succeed, "Initialize"}, + {201, &INx2VideoSession::Succeed, "InitializeWithFlag"}, + {202, &INx2VideoSession::Succeed, "InitializeWithFlagAndValue"}, {207, &INx2VideoSession::Flush, "Stop"}, + {210, &INx2VideoSession::Succeed, "Unknown210"}, {212, &INx2VideoSession::UnregisterBuffer, "UnregisterBuffer"}, {213, &INx2VideoSession::GetOutputInfo, "GetOutputInfo"}, {311, &INx2VideoSession::RegisterBuffer, "RegisterBuffer"}, {314, &INx2VideoSession::ReleaseBuffer, "ReleaseBuffer"}, + {400, &INx2VideoSession::SubmitInput, "SubmitInputBufferWithInfo"}, }); } @@ -2061,9 +2518,11 @@ private: void SubmitInput(HLERequestContext& ctx) { const auto* w = ctx.CommandBuffer(); - // Words 2-4 are the A descriptor of the compressed frame. - const u64 pts = u64{w[5]} | (u64{w[6]} << 32); - const bool end = (w[7] & 0xFF) != 0; + // Words 2-4 are the A descriptor of the compressed frame (400: 5-7 that of its information + // block), the data words follow. + const size_t words = 2 + 3 * ctx.BufferDescriptorA().size(); + const u64 pts = u64{w[words]} | (u64{w[words + 1]} << 32); + const bool end = (w[words + 2] & 0xFF) != 0; Log(ctx, "submit"); Input input{{}, pts, end}; if (!ctx.BufferDescriptorA().empty()) { @@ -2221,6 +2680,7 @@ private: const Decoded& frame, std::vector<u8>& bytes) { u64 address{}; Nx2DecoderImageLayout layout{}; + bool block_linear{}; { std::scoped_lock lk{m_objects->mutex}; const auto image = m_objects->images.find(image_id); @@ -2228,6 +2688,8 @@ private: return {m_width, m_height}; } layout = Nx2CalculateDecoderImageLayout(image->second.preference.data()); + block_linear = image->second.gpu_read && + layout.format != Tegra::Host1x::Nx2PictureFormat::Nv12; const auto memory = m_objects->memories.find(image->second.memory); if (memory != m_objects->memories.end() && memory->second.address != 0) { address = memory->second.address + image->second.offset; @@ -2241,10 +2703,25 @@ private: const u32 width = std::min(frame.picture->width, layout.width); const u32 height = std::min(frame.picture->height, layout.height); bytes.resize(layout.size); + auto& memory = system.ApplicationMemory(); + if (block_linear) { + // Read on the GPU: RGBA (the blit's format 0xd5) in the native block-linear layout, + // 16-GOB blocks, as our Eden tree writes these images. + decoder->Render(*frame.picture, Tegra::Host1x::Nx2PictureFormat::Rgba8, width, height, + layout.pitch, 0, bytes); + m_tiled.resize(layout.size); + Tegra::Texture::SwizzleTexture(m_tiled, bytes, 4, static_cast<u32>(layout.pitch / 4), + layout.height, 1, 4, 0, 1, true); + memory.WriteBlock(address, m_tiled.data(), m_tiled.size()); + if (const auto device = + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address)) { + system.GPU().InvalidateRegion(*device, m_tiled.size()); + } + return {width, height}; + } decoder->Render(*frame.picture, layout.format, width, height, layout.pitch, layout.chroma_offset, bytes); DumpFrame(bytes, layout, width, height); - auto& memory = system.ApplicationMemory(); memory.WriteBlock(address, bytes.data(), layout.pitch * height); if (layout.format == Tegra::Host1x::Nx2PictureFormat::Nv12) { memory.WriteBlock(address + layout.chroma_offset, bytes.data() + layout.chroma_offset, @@ -2290,6 +2767,7 @@ private: u64 m_submitted{}; u32 m_syncpoint{NoSyncpoint}; ///< Counts finished frames (allocated at the first register) u32 m_sync_base{}; ///< Its value before the first frame + std::vector<u8> m_tiled; ///< Decode thread: block-linear copy of a frame (WriteImage) // Shared with the decode thread. std::mutex m_mutex; @@ -2333,6 +2811,7 @@ public: {429, &INx2GraphicsSystemSession::CreateNativeImage, "CreateNativeImage"}, {430, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyNativeImage"}, {426, &INx2GraphicsSystemSession::QueryNativeImage, "MapNativeImage"}, + {427, &INx2GraphicsSystemSession::UnmapNativeImage, "UnmapNativeImage"}, {428, &INx2GraphicsSystemSession::QueryNativeImage, "QueryNativeImage"}, {435, &INx2GraphicsSystemSession::CalculateNativeImageSize, "CalculateNativeImageSize"}, {1402, &INx2GraphicsSystemSession::OpenNvsrvSession, "OpenNvsrvSession (guessed)"}, @@ -2341,6 +2820,7 @@ public: "CalculateDecoderImagePoolBuffer"}, {901, &INx2GraphicsSystemSession::Stub, "VideoInitialize (guessed)"}, {902, &INx2GraphicsSystemSession::OpenVideoSession, "OpenVideoDecoderSession (guessed)"}, + {905, &INx2GraphicsSystemSession::OpenVideoSession, "OpenVideoDecoderSessionWithFlag"}, {906, &INx2GraphicsSystemSession::OpenVideoSession, "OpenVideoDecoderSessionWithVersion"}, {1001, &INx2GraphicsSystemSession::Stub, "VideoInitialize2 (guessed)"}, {1002, &INx2GraphicsSystemSession::OpenVideoSession, "OpenVideoDecoderSession2 (guessed)"}, @@ -2353,7 +2833,92 @@ public: {1291, &INx2GraphicsSystemSession::CalculateChatImagePoolSize, "CalculateChatImagePoolSize (guessed)"}, {911, &INx2GraphicsSystemSession::CreateDecoderImage, "CreateDecoderImage"}, + // 912(u32 image id): the video group's x12, freeing one decoder output image (Switch + // Sports frees its four when its movie ends). Unlisting it stops the decode thread + // writing frames into memory the game reuses. + {912, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyDecoderImage"}, {961, &INx2GraphicsSystemSession::GetCpuImageDescriptor, "GetCpuUserImageDescriptor"}, + // Firmware commands without a known meaning: zeroed replies. + {11, &INx2GraphicsSystemSession::RegisterClientHandle, "RegisterClientHandle"}, + {113, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown113"}, + {115, &INx2GraphicsSystemSession::QueryGrantedFeatures, "QueryGrantedFeatures"}, + {120, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown120"}, + {121, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown121"}, + {122, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown122"}, + {123, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown123"}, + {124, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown124"}, + {125, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown125"}, + {126, &INx2GraphicsSystemSession::GetSessionFlag, "GetSessionFlag"}, + {131, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown131"}, + {133, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown133"}, + {187, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown187"}, + {201, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission201"}, + {211, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown211"}, + {212, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown212"}, + {213, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown213"}, + {214, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown214"}, + {215, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown215"}, + {301, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission301"}, + {303, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown303"}, + {311, &INx2GraphicsSystemSession::CreateNativeImage, "CreateFeatureImagePool311"}, + {312, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyFeatureImagePool312"}, + {321, &INx2GraphicsSystemSession::CreateNativeImage, "CreateFeatureImagePool321"}, + {361, &INx2GraphicsSystemSession::GetCpuImageDescriptor, "GetFeatureCpuImageDescriptor361"}, + {391, &INx2GraphicsSystemSession::CalculateChatImagePoolSize, "CalculateFeatureImagePoolSize391"}, + {401, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission401"}, + {402, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission402"}, + {403, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission403"}, + {424, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown424"}, + {425, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown425"}, + {427, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown427"}, + {434, &INx2GraphicsSystemSession::AttachGpuAddressSpace, "AttachGpuAddressSpace"}, + {436, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown436"}, + {461, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown461"}, + {462, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown462"}, + {463, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown463"}, + {464, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown464"}, + {465, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown465"}, + {466, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown466"}, + {467, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown467"}, + {468, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown468"}, + {493, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown493"}, + {494, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission494"}, + {603, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown603"}, + {612, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyFeatureImagePool612"}, + {613, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown613"}, + {614, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown614"}, + {615, &INx2GraphicsSystemSession::CreateNativeImage, "CreateFeatureImagePool615"}, + {691, &INx2GraphicsSystemSession::CalculateChatImagePoolSize, "CalculateFeatureImagePoolSize691"}, + {801, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission801"}, + {802, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown802"}, + {803, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown803"}, + {811, &INx2GraphicsSystemSession::CreateNativeImage, "CreateFeatureImagePool811"}, + {812, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyFeatureImagePool812"}, + {813, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown813"}, + {814, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown814"}, + {861, &INx2GraphicsSystemSession::GetCpuImageDescriptor, "GetFeatureCpuImageDescriptor861"}, + {891, &INx2GraphicsSystemSession::CalculateChatImagePoolSize, "CalculateFeatureImagePoolSize891"}, + {1003, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1003"}, + {1011, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1011"}, + {1012, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1012"}, + {1091, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1091"}, + {1101, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission1101"}, + {1102, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission1102"}, + {1111, &INx2GraphicsSystemSession::SetFeatureVersion, "SetFeatureVersion"}, + {1112, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1112"}, + {1113, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1113"}, + {1121, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1121"}, + {1203, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1203"}, + {1212, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyFeatureImagePool1212"}, + {1501, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission1501"}, + {1502, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission1502"}, + {1503, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission1503"}, + {1601, &INx2GraphicsSystemSession::CheckPermission, "CheckPermission1601"}, + {1602, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1602"}, + {1603, &INx2GraphicsSystemSession::StubReplyZeroed, "Unknown1603"}, + {1611, &INx2GraphicsSystemSession::CreateNativeImage, "CreateFeatureImagePool1611"}, + {1612, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyFeatureImagePool1612"}, + {1691, &INx2GraphicsSystemSession::CalculateChatImagePoolSize, "CalculateFeatureImagePoolSize1691"}, }); } @@ -2369,6 +2934,56 @@ public: } private: + // Feature permission checks (201, 301, 401-403, 1101, 1102, 1501-1503): each tests one bit of + // the permissions the session was opened with and fails when it is clear. Games only ask for + // features they are granted. + void CheckPermission(HLERequestContext& ctx) { + LOG_DEBUG(Service, "nx2:gsv TIPC {} permission check", ctx.GetCommand()); + TipcReplyWords(ctx, {}); + } + + // 115(u64 requested feature flags): the subset that is granted, which is all of them here. + // A zeroed reply would deny every feature. + void QueryGrantedFeatures(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + LOG_INFO(Service, "nx2:gsv TIPC 115 features {:#x}", u64{w[2]} | (u64{w[3]} << 32)); + TipcReplyWords(ctx, {w[2], w[3]}); + } + + // 126: one bit of the session's state (meaning unknown); clear. + void GetSessionFlag(HLERequestContext& ctx) { + TipcReplyWords(ctx, {0}); + } + + // 434(u32 id from nx2:gpu 33): the session takes a reference on that GPU address space. + void AttachGpuAddressSpace(HLERequestContext& ctx) { + const u32 id = ctx.CommandBuffer()[2]; + LOG_INFO(Service, "nx2:gsv TIPC 434 address space {:#x}", id); + m_address_spaces.insert(id); + TipcReplyWords(ctx, {}); + } + + // 11(copy handle): handed over once per session and kept; a second call fails. + void RegisterClientHandle(HLERequestContext& ctx) { + if (m_client_handle_registered) { + LOG_WARNING(Service, "nx2:gsv TIPC 11 called twice"); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultUnknown); + return; + } + m_client_handle_registered = true; + TipcReplyWords(ctx, {}); + } + + // 1111(u32): a version or mode number the server checks and records. + void SetFeatureVersion(HLERequestContext& ctx) { + LOG_INFO(Service, "nx2:gsv TIPC 1111 value {:#x}", ctx.CommandBuffer()[2]); + TipcReplyWords(ctx, {}); + } + + std::set<u32> m_address_spaces; ///< Ids attached with 434 + bool m_client_handle_registered = false; ///< Set by 11 + // Index of the first raw data word of a TIPC request: header (2 words), then an optional // special header with pid and copy/move handle words. static size_t TipcDataOffset(const u32* cmd_buf) { @@ -2392,7 +3007,9 @@ private: // and InitializeVp9Decoder failed with 2001-0114 (title 1 aborts when a movie starts). // Newer SDKs (nn::codec::video::detail::InitializeServerContext) send 906(u8, u64 decoder // library version) instead of 902; title 4 sends version 0x3'00030000 and aborted with - // InvalidHandle at its first movie while 906 was auto-stubbed. + // InvalidHandle at its first movie while 906 was auto-stubbed. The SDK of later titles + // (ipc::Initialize(bool)) sends 905(u8) instead of 902 when its flag is set; the reply is the + // same. 1001/1002 open the encoder's creator session (CreateVideoEncoderService uses it). void OpenVideoSession(HLERequestContext& ctx) { const auto* w = ctx.CommandBuffer(); const bool second = ctx.GetCommand() == 1002; @@ -2405,13 +3022,11 @@ private: // nn::camera::sdk::CreateCameraContext sends 601() then 602(u64 1), which returns the camera // context session (handle at word 3, result at word 4). Auto-stubbed, 602 had no handle, the - // SDK sent 603 (destroy) and title 1 (1.8.0) aborted with 2034-0000. Its commands fall - // through to the nx2 auto stub. + // SDK sent 603 (destroy) and title 1 (1.8.0) aborted with 2034-0000. Its commands are + // answered by INx2CameraSession. void OpenCameraSession(HLERequestContext& ctx) { LOG_WARNING(Service, "(STUBBED) nx2:gsv TIPC 602, returning a nx2:camera session"); - ReturnRawSession(system, ctx, - std::make_shared<INx2StubInterface>( - system, INx2StubInterface::InternName("nx2:camera"))); + ReturnRawSession(system, ctx, std::make_shared<INx2CameraSession>(system)); } // nn::chat::gsv::Initialize sends 1201 then 1202, the same pair as 901/902 and 1001/1002. @@ -2597,8 +3212,19 @@ private: void DestroyNativeMemory(HLERequestContext& ctx) { const u32 id = ctx.CommandBuffer()[2]; - std::scoped_lock lk{m_objects->mutex}; - m_objects->memories.erase(id); + std::vector<std::optional<std::pair<u64, u64>>> unmaps; + { + std::scoped_lock lk{m_objects->mutex}; + for (auto& [image_id, image] : m_objects->images) { + if (image.memory == id && image.mapped_here) { + unmaps.push_back(TakeDecoderMapping(image)); + } + } + m_objects->memories.erase(id); + } + for (const auto& range : unmaps) { + DropDecoderMapping(range); + } LOG_DEBUG(Service, "nx2:gsv 112 DestroyNativeMemory id {}", id); TipcReplyWords(ctx, {}); } @@ -2648,6 +3274,8 @@ private: TipcReplyWords(ctx, {id}); } + // Pool size of a feature's images (1291 here; 391, 691, 891 and 1691 for the other feature + // groups, which share the server template). // nn::chat::gsv::CalculateRequiredBufferMemory: 1291(preference[16], u32 count), the request // of 1510, replies result, u64 size, u64 alignment. Auto-stubbed, the size became 0x1000 and // the camera pool (1211) got memory for none of its images. @@ -2673,6 +3301,9 @@ private: // the 429 preference words (0x10, 0x14), a guess. void QueryNativeImage(HLERequestContext& ctx) { const u32 handle = ctx.CommandBuffer()[2]; + if (QueryDecoderImage(ctx, handle)) { + return; + } std::array<u32, 16> preference{}; u64 image_offset = 0; u64 memory_address = 0; @@ -2717,6 +3348,108 @@ private: } } + // 428/426 for a decoder output image (Fire Emblem: Fortune's Weave's movie player blits each + // one, block-linear RGBA, format 0xd5, block height 16 GOBs, into a texture of its own). The + // reply follows our Eden tree, where the movies play: words 3-4 the GPU address, 5-6 the last + // byte's address, then the descriptor {1, 0, dimensions, .., 0x01000001, 1, offset, size} + // (offset 0 once mapped). 426 maps the image at its own address for the GPU: decoder pool + // memory is process memory (114) that no GPU mapping covers, and the blit read zeros. + bool QueryDecoderImage(HLERequestContext& ctx, u32 handle) { + const bool map = ctx.GetCommand() == 426; + u64 address{}; + u64 size{}; + u64 offset{}; + u32 dimensions{}; + bool register_mapping = false; + { + std::scoped_lock lk{m_objects->mutex}; + const auto image = m_objects->images.find(handle); + if (image == m_objects->images.end() || !image->second.decoder) { + return false; + } + const auto memory = m_objects->memories.find(image->second.memory); + if (memory == m_objects->memories.end() || memory->second.address == 0) { + return false; + } + image->second.gpu_read = true; + offset = image->second.offset; + size = image->second.size; + dimensions = image->second.preference[2]; + if (map) { + address = memory->second.address + offset; + register_mapping = image->second.gpu_maps++ == 0 && + !Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address); + image->second.mapped_here |= register_mapping; + } + } + if (register_mapping) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Register(address, size); + } + LOG_INFO(Service, "nx2:gsv {} decoder image {} at {:#x}+{:#x}{}", ctx.GetCommand(), handle, + address, size, register_mapping ? ", mapped for the GPU" : ""); + std::array<u32, 37> words{}; + if (map) { + const u64 last = address + size - 1; + words[0] = static_cast<u32>(address); + words[1] = static_cast<u32>(address >> 32); + words[2] = static_cast<u32>(last); + words[3] = static_cast<u32>(last >> 32); + } + u32* const desc = words.data() + 4; + desc[0] = 1; + desc[2] = dimensions; + desc[8] = 0x01000001; + desc[9] = 1; + desc[10] = map ? 0 : static_cast<u32>(offset); + desc[11] = static_cast<u32>(size); + IPC::ResponseBuilder rb{ctx, static_cast<u32>(2 + words.size())}; + rb.PushRaw<u32>(ResultSuccess.raw); + for (const u32 w : words) { + rb.Push<u32>(w); + } + return true; + } + + // 427(u32 handle, u32 address space, u64 address, u64 last address): undoes a 426. The last + // unmap of a decoder image drops the GPU mapping 426 made. + void UnmapNativeImage(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u32 handle = w[2]; + std::optional<std::pair<u64, u64>> unmap; + { + std::scoped_lock lk{m_objects->mutex}; + if (const auto image = m_objects->images.find(handle); + image != m_objects->images.end() && image->second.gpu_maps != 0 && + --image->second.gpu_maps == 0) { + unmap = TakeDecoderMapping(image->second); + } + } + LOG_DEBUG(Service, "nx2:gsv 427 image {} [{:#x}, {:#x}]", handle, Pair(w, 4), Pair(w, 6)); + DropDecoderMapping(unmap); + TipcReplyWords(ctx, {}); + } + + // The range 426 mapped for an image, if it did; called with m_objects->mutex held. + std::optional<std::pair<u64, u64>> TakeDecoderMapping(Nx2NativeObjects::Image& image) { + image.gpu_maps = 0; + if (!std::exchange(image.mapped_here, false)) { + return std::nullopt; + } + const auto memory = m_objects->memories.find(image.memory); + if (memory == m_objects->memories.end() || memory->second.address == 0) { + return std::nullopt; + } + return std::pair{memory->second.address + image.offset, image.size}; + } + + static void DropDecoderMapping(const std::optional<std::pair<u64, u64>>& range) { + if (range) { + auto& registry = Nvidia::Devices::Nx2GpuMemoryRegistry::Get(); + registry.Unmap(range->first, range->second); + registry.ReleaseBacking(range->first, range->second); + } + } + // 911(preference[16], u32 count, u32 memory id) (nn::codec::InitializeDecoderImagePool): // result, the id of the first of `count` decoder output images; the client names image i of // the pool id + i. They share the pool's memory, one image size apart. @@ -2775,8 +3508,15 @@ private: void DestroyNativeImage(HLERequestContext& ctx) { const u32 id = ctx.CommandBuffer()[2]; - std::scoped_lock lk{m_objects->mutex}; - m_objects->images.erase(id); + std::optional<std::pair<u64, u64>> unmap; + { + std::scoped_lock lk{m_objects->mutex}; + if (const auto image = m_objects->images.find(id); image != m_objects->images.end()) { + unmap = TakeDecoderMapping(image->second); + m_objects->images.erase(image); + } + } + DropDecoderMapping(unmap); TipcReplyWords(ctx, {}); } diff --git a/src/core/hle/service/ro/ro.cpp b/src/core/hle/service/ro/ro.cpp index 335efa3ad3..2af4f1cf58 100644 --- a/src/core/hle/service/ro/ro.cpp +++ b/src/core/hle/service/ro/ro.cpp @@ -356,6 +356,22 @@ public: // Read NRR. NrrHeader header{}; process->GetMemory().ReadBlock(nrr_address, std::addressof(header), sizeof(header)); + size_t hashes_offset = header.GetHashesOffset(); + u32 num_hashes = header.GetNumHashes(); + if (process->IsSwitch2()) { + // Switch 2 NRRs carry 3072-bit keys and signatures: the fields after them (program + // id, size, kind, hash list offset and count) sit 0x180 bytes later, the hash list + // offset and count at 0x4c0. + std::array<u32, 2> fields{}; + process->GetMemory().ReadBlock(nrr_address + 0x4c0, fields.data(), sizeof(fields)); + hashes_offset = fields[0]; + num_hashes = fields[1]; + } + // The hash list lies inside the NRR (read as a Switch 1 header, a Switch 2 one asked for + // 51 GB of hashes). + R_UNLESS(hashes_offset <= nrr_size && + num_hashes <= (nrr_size - hashes_offset) / sizeof(Sha256Hash), + RO::ResultInvalidNrr); // Set NRR info. context->SetNrrInfoInUse(nrr_info, true); @@ -363,10 +379,9 @@ public: nrr_info->nrr_heap_size = nrr_size; // Read NRR hash list. - nrr_info->hashes.resize(header.GetNumHashes()); - process->GetMemory().ReadBlock(nrr_address + header.GetHashesOffset(), - nrr_info->hashes.data(), - sizeof(Sha256Hash) * header.GetNumHashes()); + nrr_info->hashes.resize(num_hashes); + process->GetMemory().ReadBlock(nrr_address + hashes_offset, nrr_info->hashes.data(), + sizeof(Sha256Hash) * num_hashes); R_SUCCEED(); } diff --git a/src/core/hle/service/service.cpp b/src/core/hle/service/service.cpp index 1490af0328..b47d67468b 100644 --- a/src/core/hle/service/service.cpp +++ b/src/core/hle/service/service.cpp @@ -9,6 +9,10 @@ #include <tuple> #include <mutex> #include <map> +#include <memory> +#include <set> +#include <string> +#include <string_view> #include <chrono> #include <algorithm> #include <cstdlib> @@ -18,8 +22,10 @@ #include "common/settings.h" #include "core/core.h" #include "core/hle/ipc.h" +#include "core/hle/kernel/k_event.h" #include "core/hle/kernel/kernel.h" #include "core/hle/service/ipc_helpers.h" +#include "core/hle/service/kernel_helpers.h" #include "core/hle/service/service.h" #include "core/hle/service/nx2_stub_interface.h" #include "core/hle/service/sm/sm.h" @@ -49,6 +55,75 @@ ServiceFrameworkBase::ServiceFrameworkBase(Core::System& system_, const char* se ServiceFrameworkBase::~ServiceFrameworkBase() { // Wait for other threads to release access before destroying const auto guard = ServiceFrameworkBase::LockService(); + if (stub_event) { + stub_context->CloseEvent(stub_event); + } +} + +namespace { + +// Callers read the reply words they expect without checking how many were written, so an empty +// reply hands them the request's leftover words as data (an HDR output format, a refresh rate, a +// device id). Stub replies carry this many zeroed words. +constexpr u32 StubZeroedWords = 32; + +void PushStubZeroes(IPC::ResponseBuilder& rb) { + for (u32 i = 0; i < StubZeroedWords; ++i) { + rb.Push<u32>(0); + } +} + +} // namespace + +bool ServiceFrameworkBase::StubAllowed(HLERequestContext& ctx) { + static const bool env_auto_stub = std::getenv("EDEN_NX2_AUTO_STUB") != nullptr; + const auto* const caller = ctx.GetThread().GetOwnerProcess(); + if (Settings::values.use_auto_stub || env_auto_stub || + std::string_view{service_name}.starts_with(Nx2StubPrefix) || + (caller != nullptr && caller->IsSwitch2())) { + // Logged once per command: these are still unimplemented. + static std::mutex mutex; + static std::set<std::pair<std::string_view, u32>> logged; + std::scoped_lock lk{mutex}; + if (logged.emplace(service_name, ctx.GetCommand()).second) { + LOG_WARNING(Service, "(STUBBED) {}::{} (placeholder reply)", service_name, ctx.GetCommand()); + } + return true; + } + const auto& table = ctx.IsTipc() ? handlers_tipc : handlers; + const auto it = table.find(ctx.GetCommand()); + ReportUnimplementedFunction(ctx, it == table.end() ? nullptr : &it->second); + return false; +} + +void ServiceFrameworkBase::StubReplyZeroed(HLERequestContext& ctx) { + if (!StubAllowed(ctx)) { + return; + } + IPC::ResponseBuilder rb{ctx, 2 + StubZeroedWords}; + rb.Push(ResultSuccess); + PushStubZeroes(rb); +} + +void ServiceFrameworkBase::StubReplyEvent(HLERequestContext& ctx) { + if (!StubAllowed(ctx)) { + return; + } + if (!stub_event) { + stub_context = std::make_unique<KernelHelpers::ServiceContext>(system, service_name); + stub_event = stub_context->CreateEvent(fmt::format("{}:StubEvent", service_name)); + } + IPC::ResponseBuilder rb{ctx, 2 + StubZeroedWords, 1, 0}; + rb.Push(ResultSuccess); + PushStubZeroes(rb); + rb.PushCopyObjects(ctx, stub_event->GetReadableEvent()); +} + +void ServiceFrameworkBase::StubReplyInterface(HLERequestContext& ctx) { + if (!StubAllowed(ctx)) { + return; + } + ReturnNx2StubInterface(system, ctx, service_name); } void ServiceFrameworkBase::RegisterHandlersBase(const FunctionInfoBase* functions, std::size_t n) { @@ -67,6 +142,27 @@ void ServiceFrameworkBase::RegisterHandlersBaseTipc(const FunctionInfoBase* func void ServiceFrameworkBase::ReportUnimplementedFunction(HLERequestContext& ctx, const FunctionInfoBase* info) { + // EDEN_NX2_AUTO_STUB=1 auto-stubs every unknown command without touching the config. Switch 2 + // programs are always auto-stubbed: they still call many commands nobody has implemented. + static const bool env_auto_stub = std::getenv("EDEN_NX2_AUTO_STUB") != nullptr; + const auto* const caller = ctx.GetThread().GetOwnerProcess(); + const bool auto_stub = Settings::values.use_auto_stub || env_auto_stub || + std::string_view{service_name}.starts_with(Nx2StubPrefix) || + (caller != nullptr && caller->IsSwitch2()); + // Auto-stubbed commands are reported once each: title 1 polls nx2:camera 29 every frame + // (~47 calls a second), and every report wrote a Critical line, which flushes the log file. + if (auto_stub) { + static std::mutex reported_mutex; + static std::set<std::pair<std::string, u32>> reported; + std::scoped_lock lk{reported_mutex}; + if (!reported.emplace(service_name, ctx.GetCommand()).second) { + IPC::ResponseBuilder rb{ctx, 2 + StubZeroedWords}; + rb.Push(ResultSuccess); + PushStubZeroes(rb); + return; + } + } + auto cmd_buf = ctx.CommandBuffer(); std::string function_name = info == nullptr ? "<unknown>" : info->name; @@ -80,16 +176,11 @@ void ServiceFrameworkBase::ReportUnimplementedFunction(HLERequestContext& ctx, system.GetReporter().SaveUnimplementedFunctionReport(ctx, ctx.GetCommand(), function_name, service_name); UNIMPLEMENTED_MSG("Unknown / unimplemented {}", fmt::to_string(buf)); - // EDEN_NX2_AUTO_STUB=1 auto-stubs every unknown command without touching the config. Switch 2 - // programs are always auto-stubbed: they still call many commands nobody has implemented. - static const bool env_auto_stub = std::getenv("EDEN_NX2_AUTO_STUB") != nullptr; - const auto* const caller = ctx.GetThread().GetOwnerProcess(); - if (Settings::values.use_auto_stub || env_auto_stub || - std::string_view{service_name}.starts_with(Nx2StubPrefix) || - (caller != nullptr && caller->IsSwitch2())) { - LOG_WARNING(Service, "Using auto stub fallback!"); - IPC::ResponseBuilder rb{ctx, 2}; + if (auto_stub) { + LOG_WARNING(Service, "Using auto stub fallback (repeats of this command are not logged)"); + IPC::ResponseBuilder rb{ctx, 2 + StubZeroedWords}; rb.Push(ResultSuccess); + PushStubZeroes(rb); } } diff --git a/src/core/hle/service/service.h b/src/core/hle/service/service.h index 68b0d636d8..d1be62c8c7 100644 --- a/src/core/hle/service/service.h +++ b/src/core/hle/service/service.h @@ -7,6 +7,7 @@ #pragma once #include <cstddef> +#include <memory> #include <mutex> #include "common/container/unordered_map.h" #include "common/common_types.h" @@ -20,12 +21,17 @@ class System; } namespace Kernel { +class KEvent; class KServerSession; class ServiceThread; } // namespace Kernel namespace Service { +namespace KernelHelpers { +class ServiceContext; +} + namespace FileSystem { class FileSystemController; } @@ -77,6 +83,24 @@ protected: [[nodiscard]] virtual std::unique_lock<std::mutex> LockService() noexcept { return std::unique_lock{lock_service}; } + + /// Placeholder handlers for commands whose reply shape is known but whose meaning is not, for + /// Switch 2 callers (others get the unimplemented-function path, as with nullptr): + /// StubReplyZeroed replies with zeroed data words, StubReplyEvent adds a copy handle to an + /// event that never fires, and StubReplyInterface returns an nx2: stub interface whose + /// commands are auto-stubbed. + void StubReplyZeroed(HLERequestContext& ctx); + void StubReplyEvent(HLERequestContext& ctx); + void StubReplyInterface(HLERequestContext& ctx); + +private: + /// False (after reporting the command as unimplemented) for callers the stubs don't serve. + bool StubAllowed(HLERequestContext& ctx); + + /// Event handed out by StubReplyEvent, created on first use. + Kernel::KEvent* stub_event{}; + std::unique_ptr<KernelHelpers::ServiceContext> stub_context; + private: template <typename T> friend class ServiceFramework; diff --git a/src/core/hle/service/set/settings_server.cpp b/src/core/hle/service/set/settings_server.cpp index 942dddc9bc..78847ba082 100644 --- a/src/core/hle/service/set/settings_server.cpp +++ b/src/core/hle/service/set/settings_server.cpp @@ -102,9 +102,14 @@ ISettingsServer::ISettingsServer(Core::System& system_) : ServiceFramework{syste {7, C<&ISettingsServer::GetKeyCodeMap>, "GetKeyCodeMap"}, {8, C<&ISettingsServer::GetQuestFlag>, "GetQuestFlag"}, {9, C<&ISettingsServer::GetKeyCodeMap2>, "GetKeyCodeMap2"}, - {10, nullptr, "GetFirmwareVersionForDebug"}, + {10, &ISettingsServer::StubReplyZeroed, "GetFirmwareVersionForDebug"}, {11, C<&ISettingsServer::GetDeviceNickName>, "GetDeviceNickName"}, {12, C<&ISettingsServer::GetKeyCodeMapByPort>, "GetKeyCodeMapByPort"}, + {13, &ISettingsServer::StubReplyZeroed, "Unknown13"}, + {14, &ISettingsServer::StubReplyZeroed, "Unknown14"}, + {15, &ISettingsServer::StubReplyZeroed, "Unknown15"}, + {16, &ISettingsServer::StubReplyZeroed, "Unknown16"}, + {17, &ISettingsServer::StubReplyZeroed, "Unknown17"}, }; // clang-format on diff --git a/src/core/hle/service/sockets/bsd.cpp b/src/core/hle/service/sockets/bsd.cpp index 0137a3be66..02c6bb7762 100644 --- a/src/core/hle/service/sockets/bsd.cpp +++ b/src/core/hle/service/sockets/bsd.cpp @@ -1089,10 +1089,10 @@ BSD_USA::BSD_USA(Core::System& system_, const char* name, bool is_user_) {1, &BSD_USA::StartMonitoring, "StartMonitoring"}, {2, &BSD_USA::Socket, "Socket"}, {3, &BSD_USA::SocketExempt, "SocketExempt"}, - {4, nullptr, "Open"}, + {4, &BSD_USA::StubReplyZeroed, "Open"}, {5, &BSD_USA::Select, "Select"}, {6, &BSD_USA::Poll, "Poll"}, - {7, nullptr, "Sysctl"}, + {7, &BSD_USA::StubReplyZeroed, "Sysctl"}, {8, &BSD_USA::Recv, "Recv"}, {9, &BSD_USA::RecvFrom, "RecvFrom"}, {10, &BSD_USA::Send, "Send"}, @@ -1104,33 +1104,37 @@ BSD_USA::BSD_USA(Core::System& system_, const char* name, bool is_user_) {16, &BSD_USA::GetSockName, "GetSockName"}, {17, &BSD_USA::GetSockOpt, "GetSockOpt"}, {18, &BSD_USA::Listen, "Listen"}, - {19, nullptr, "Ioctl"}, + {19, &BSD_USA::StubReplyZeroed, "Ioctl"}, {20, &BSD_USA::Fcntl, "Fcntl"}, {21, &BSD_USA::SetSockOpt, "SetSockOpt"}, {22, &BSD_USA::Shutdown, "Shutdown"}, - {23, nullptr, "ShutdownAllSockets"}, + {23, &BSD_USA::StubReplyZeroed, "ShutdownAllSockets"}, {24, &BSD_USA::Write, "Write"}, {25, &BSD_USA::Read, "Read"}, {26, &BSD_USA::Close, "Close"}, {27, &BSD_USA::DuplicateSocket, "DuplicateSocket"}, - {28, nullptr, "GetResourceStatistics"}, - {29, nullptr, "RecvMMsg"}, //3.0.0+ - {30, nullptr, "SendMMsg"}, //3.0.0+ + {28, &BSD_USA::StubReplyZeroed, "GetResourceStatistics"}, + {29, &BSD_USA::StubReplyZeroed, "RecvMMsg"}, //3.0.0+ + {30, &BSD_USA::StubReplyZeroed, "SendMMsg"}, //3.0.0+ {31, &BSD_USA::EventFd, "EventFd"}, //7.0.0+ - {32, nullptr, "RegisterResourceStatisticsName"}, //7.0.0+ - {33, nullptr, "RegisterClientShared"}, //10.0.0+ - {34, nullptr, "GetSocketStatistics"}, //15.0.0+ - {35, nullptr, "NifIoctl"}, //17.0.0+ - {36, nullptr, "Unknown36"}, //18.0.0+ - {37, nullptr, "Unknown37"}, //18.0.0+ - {38, nullptr, "Unknown38"}, //18.0.0+ - {39, nullptr, "Unknown39"}, //20.0.0+ - {40, nullptr, "Unknown40"}, //20.0.0+ - {41, nullptr, "Unknown41"}, //21.0.0+ - {42, nullptr, "Unknown42"}, //21.0.0+ - {43, nullptr, "Unknown43"}, //21.0.0+ - {200, nullptr, "SetThreadCoreMask"}, //15.0.0+ - {201, nullptr, "GetThreadCoreMask"}, //15.0.0+ + {32, &BSD_USA::StubReplyZeroed, "RegisterResourceStatisticsName"}, //7.0.0+ + {33, &BSD_USA::StubReplyZeroed, "RegisterClientShared"}, //10.0.0+ + {34, &BSD_USA::StubReplyZeroed, "GetSocketStatistics"}, //15.0.0+ + {35, &BSD_USA::StubReplyZeroed, "NifIoctl"}, //17.0.0+ + // Switch 2 titles register with 36 (16 bytes and the pid; a bsd errno back) instead of + // RegisterClient. The auto stub left the errno to the message buffer's leftovers, and + // Minecraft's socket layer failed later in EventFd. + {36, &BSD_USA::RegisterClient, "RegisterClientSwitch2"}, //18.0.0+ + {37, &BSD_USA::StubReplyZeroed, "Unknown37"}, //18.0.0+ + {38, &BSD_USA::StubReplyZeroed, "Unknown38"}, //18.0.0+ + {39, &BSD_USA::StubReplyZeroed, "Unknown39"}, //20.0.0+ + {40, &BSD_USA::StubReplyZeroed, "Unknown40"}, //20.0.0+ + {41, &BSD_USA::StubReplyEvent, "Unknown41"}, //21.0.0+ + {42, &BSD_USA::StubReplyZeroed, "Unknown42"}, //21.0.0+ + {43, &BSD_USA::StubReplyZeroed, "Unknown43"}, //21.0.0+ + {200, &BSD_USA::StubReplyZeroed, "SetThreadCoreMask"}, //15.0.0+ + {201, &BSD_USA::StubReplyZeroed, "GetThreadCoreMask"}, //15.0.0+ + {44, &BSD_USA::StubReplyZeroed, "Unknown44"}, }; // clang-format on @@ -1158,22 +1162,28 @@ BSDCFG::BSDCFG(Core::System& system_, const char *name) : ServiceFramework{system_, name} { // clang-format off static const FunctionInfo functions[] = { - {0, nullptr, "SetIfUp"}, - {1, nullptr, "SetIfUpWithEvent"}, - {2, nullptr, "CancelIf"}, - {3, nullptr, "SetIfDown"}, - {4, nullptr, "GetIfState"}, - {5, nullptr, "DhcpRenew"}, - {6, nullptr, "AddStaticArpEntry"}, - {7, nullptr, "RemoveArpEntry"}, - {8, nullptr, "LookupArpEntry"}, - {9, nullptr, "LookupArpEntry2"}, - {10, nullptr, "ClearArpEntries"}, - {11, nullptr, "ClearArpEntries2"}, - {12, nullptr, "PrintArpEntries"}, - {13, nullptr, "Unknown13"}, - {14, nullptr, "Unknown14"}, - {15, nullptr, "Unknown15"}, + {0, &BSDCFG::StubReplyZeroed, "SetIfUp"}, + {1, &BSDCFG::StubReplyZeroed, "SetIfUpWithEvent"}, + {2, &BSDCFG::StubReplyZeroed, "CancelIf"}, + {3, &BSDCFG::StubReplyZeroed, "SetIfDown"}, + {4, &BSDCFG::StubReplyZeroed, "GetIfState"}, + {5, &BSDCFG::StubReplyZeroed, "DhcpRenew"}, + {6, &BSDCFG::StubReplyZeroed, "AddStaticArpEntry"}, + {7, &BSDCFG::StubReplyZeroed, "RemoveArpEntry"}, + {8, &BSDCFG::StubReplyZeroed, "LookupArpEntry"}, + {9, &BSDCFG::StubReplyZeroed, "LookupArpEntry2"}, + {10, &BSDCFG::StubReplyZeroed, "ClearArpEntries"}, + {11, &BSDCFG::StubReplyZeroed, "ClearArpEntries2"}, + {12, &BSDCFG::StubReplyZeroed, "PrintArpEntries"}, + {13, &BSDCFG::StubReplyZeroed, "Unknown13"}, + {14, &BSDCFG::StubReplyZeroed, "Unknown14"}, + {15, &BSDCFG::StubReplyZeroed, "Unknown15"}, + {16, &BSDCFG::StubReplyZeroed, "Unknown16"}, + {100, &BSDCFG::StubReplyZeroed, "Unknown100"}, + {101, &BSDCFG::StubReplyZeroed, "Unknown101"}, + {102, &BSDCFG::StubReplyZeroed, "Unknown102"}, + {103, &BSDCFG::StubReplyZeroed, "Unknown103"}, + {104, &BSDCFG::StubReplyZeroed, "Unknown104"}, }; // clang-format on diff --git a/src/core/hle/service/sockets/nsd.cpp b/src/core/hle/service/sockets/nsd.cpp index 241f2971ca..eb867cf08a 100644 --- a/src/core/hle/service/sockets/nsd.cpp +++ b/src/core/hle/service/sockets/nsd.cpp @@ -86,7 +86,7 @@ void NSD::SetChangeEnvironmentIdentifierDisabled(HLERequestContext& ctx) { LOG_WARNING(Service, "(STUBBED) called, disabled={}", disabled); - IPC::ResponseBuilder rb{ctx, 1}; + IPC::ResponseBuilder rb{ctx, 2}; rb.Push(ResultSuccess); } @@ -114,7 +114,7 @@ void NSD::ResolveEx(HLERequestContext& ctx) { } ctx.WriteBuffer(fqdn_out); - IPC::ResponseBuilder rb{ctx, 4}; + IPC::ResponseBuilder rb{ctx, 3}; rb.Push(ResultSuccess); rb.Push(ResultSuccess); } diff --git a/src/core/hle/service/ssl/ssl.cpp b/src/core/hle/service/ssl/ssl.cpp index 999efd4295..15aa478b75 100644 --- a/src/core/hle/service/ssl/ssl.cpp +++ b/src/core/hle/service/ssl/ssl.cpp @@ -87,7 +87,7 @@ public: {3, D<&ISslConnection::SetIoMode>, "SetIoMode"}, {4, D<&ISslConnection::GetSocketDescriptor>, "GetSocketDescriptor"}, {5, D<&ISslConnection::GetHostName>, "GetHostName"}, - {6, nullptr, "GetVerifyOption"}, + {6, &ISslConnection::StubReplyZeroed, "GetVerifyOption"}, {7, D<&ISslConnection::GetIoMode>, "GetIoMode"}, {8, D<&ISslConnection::DoHandshake>, "DoHandshake"}, {9, &ISslConnection::DoHandshakeGetServerCert, "DoHandshakeGetServerCert"}, @@ -105,18 +105,20 @@ public: {21, D<&ISslConnection::GetRenegotiationMode>, "GetRenegotiationMode"}, {22, D<&ISslConnection::SetOption>, "SetOption"}, {23, D<&ISslConnection::GetOption>, "GetOption"}, - {24, nullptr, "GetVerifyCertErrors"}, - {25, nullptr, "GetCipherInfo"}, + {24, &ISslConnection::StubReplyZeroed, "GetVerifyCertErrors"}, + {25, &ISslConnection::StubReplyZeroed, "GetCipherInfo"}, {26, D<&ISslConnection::SetNextAlpnProto>, "SetNextAlpnProto"}, {27, D<&ISslConnection::GetNextAlpnProto>, "GetNextAlpnProto"}, - {28, nullptr, "SetDtlsSocketDescriptor"}, - {29, nullptr, "GetDtlsHandshakeTimeout"}, - {30, nullptr, "SetPrivateOption"}, - {31, nullptr, "SetSrtpCiphers"}, - {32, nullptr, "GetSrtpCipher"}, - {33, nullptr, "ExportKeyingMaterial"}, - {34, nullptr, "SetIoTimeout"}, - {35, nullptr, "GetIoTimeout"}, + {28, &ISslConnection::StubReplyZeroed, "SetDtlsSocketDescriptor"}, + {29, &ISslConnection::StubReplyZeroed, "GetDtlsHandshakeTimeout"}, + {30, &ISslConnection::StubReplyZeroed, "SetPrivateOption"}, + {31, &ISslConnection::StubReplyZeroed, "SetSrtpCiphers"}, + {32, &ISslConnection::StubReplyZeroed, "GetSrtpCipher"}, + {33, &ISslConnection::StubReplyZeroed, "ExportKeyingMaterial"}, + {34, &ISslConnection::StubReplyZeroed, "SetIoTimeout"}, + {35, &ISslConnection::StubReplyZeroed, "GetIoTimeout"}, + {36, &ISslConnection::StubReplyZeroed, "Unknown36"}, + {37, &ISslConnection::StubReplyZeroed, "Unknown37"}, }; // clang-format on @@ -420,10 +422,14 @@ private: R_SUCCEED(); } - Result GetNextAlpnProto(OutBuffer<BufferAttr_HipcMapAlias> data, Out<u32> to_write) { - *to_write = u32((std::min)(next_alpn_proto.size(), data.size())); - next_alpn_proto.assign(data.begin(), data.begin() + *to_write); - LOG_DEBUG(Service_SSL, "GetNextAlpnProto called, size={}", *to_write); + // Replies with two words, the ALPN state and the size written to the buffer. It used to reply + // with one and copy the caller's (output) buffer over the protocols SetNextAlpnProto stored. + // No TLS negotiation happens here, so report that nothing was negotiated. + Result GetNextAlpnProto(OutBuffer<BufferAttr_HipcMapAlias> data, Out<u32> out_state, + Out<u32> out_size) { + LOG_DEBUG(Service_SSL, "GetNextAlpnProto called, buffer size={}", data.size()); + *out_state = 0; + *out_size = 0; R_SUCCEED(); } @@ -557,8 +563,10 @@ private: rb.Push(client_id); } - Result RegisterInternalPki() { + // Replies with a u64 (the registered certificate's id); the reply used to carry no data. + Result RegisterInternalPki(Out<u64> out_id) { LOG_WARNING(Service_SSL, "(STUBBED) called"); + *out_id = 0; R_SUCCEED(); } }; @@ -636,7 +644,7 @@ class ISslServiceForSystem final : public ServiceFramework<ISslServiceForSystem> explicit ISslServiceForSystem(Core::System& system_) : ServiceFramework{system_, "ssl:s"} { // clang-format off static const FunctionInfo functions[] = { - {0, D<&ISslServiceForSystem::CreateContext>, "CreateContext"}, + {0, &ISslServiceForSystem::CreateContext, "CreateContext"}, {1, D<&ISslServiceForSystem::GetContextCount>, "GetContextCount"}, {2, D<&ISslServiceForSystem::GetCertificates>, "GetCertificates"}, {3, D<&ISslServiceForSystem::GetCertificateBufSize>, "GetCertificateBufSize"}, @@ -646,7 +654,7 @@ class ISslServiceForSystem final : public ServiceFramework<ISslServiceForSystem> {7, D<&ISslServiceForSystem::SetDebugOption>, "SetDebugOption"}, {8, D<&ISslServiceForSystem::GetDebugOption>, "GetDebugOption"}, {9, D<&ISslServiceForSystem::ClearTls12FallbackFlag>, "ClearTls12FallbackFlag"}, - {100, D<&ISslServiceForSystem::CreateContextForSystem>, "CreateContextForSystem"}, + {100, &ISslServiceForSystem::CreateContext, "CreateContextForSystem"}, {101, D<&ISslServiceForSystem::SetThreadCoreMask>, "SetThreadCoreMask"}, {102, D<&ISslServiceForSystem::GetThreadCoreMask>, "GetThreadCoreMask"}, {103, D<&ISslServiceForSystem::VerifySignature>, "VerifySignature"} @@ -656,19 +664,30 @@ class ISslServiceForSystem final : public ServiceFramework<ISslServiceForSystem> RegisterHandlers(functions); }; - Result CreateContext() { - LOG_DEBUG(Service_SSL, "(STUBBED) called."); + // 0 and 100(SslVersion, padding, u64 pid placeholder; pid): an ISslContext, like ssl's + // CreateContext. The empty reply returned no object, so the caller got handle 0 + // (Minecraft calls 100). + void CreateContext(HLERequestContext& ctx) { + struct Parameters { + SslVersion ssl_version; + INSERT_PADDING_BYTES(0x4); + u64 pid_placeholder; + }; + static_assert(sizeof(Parameters) == 0x10, "Parameters is an invalid size"); - // TODO (jarrodnorwell) + IPC::RequestParser rp{ctx}; + const auto parameters = rp.PopRaw<Parameters>(); + LOG_WARNING(Service_SSL, "(STUBBED) called, command={}, api_version={}", ctx.GetCommand(), + parameters.ssl_version.api_version); - return ResultSuccess; - }; + IPC::ResponseBuilder rb{ctx, 2, 0, 1}; + rb.Push(ResultSuccess); + rb.PushIpcInterface<ISslContext>(ctx, system, parameters.ssl_version); + } - Result GetContextCount() { + Result GetContextCount(Out<u32> out_count) { LOG_DEBUG(Service_SSL, "(STUBBED) called."); - - // TODO (jarrodnorwell) - + *out_count = 0; return ResultSuccess; }; @@ -704,11 +723,9 @@ class ISslServiceForSystem final : public ServiceFramework<ISslServiceForSystem> return ResultSuccess; }; - Result FlushSessionCache() { + Result FlushSessionCache(Out<u32> out_count) { LOG_DEBUG(Service_SSL, "(STUBBED) called."); - - // TODO (jarrodnorwell) - + *out_count = 0; return ResultSuccess; }; @@ -736,14 +753,6 @@ class ISslServiceForSystem final : public ServiceFramework<ISslServiceForSystem> return ResultSuccess; }; - Result CreateContextForSystem() { - LOG_DEBUG(Service_SSL, "(STUBBED) called."); - - // TODO (jarrodnorwell) - - return ResultSuccess; - }; - Result SetThreadCoreMask() { LOG_DEBUG(Service_SSL, "(STUBBED) called."); @@ -752,11 +761,9 @@ class ISslServiceForSystem final : public ServiceFramework<ISslServiceForSystem> return ResultSuccess; }; - Result GetThreadCoreMask() { + Result GetThreadCoreMask(Out<u64> out_mask) { LOG_DEBUG(Service_SSL, "(STUBBED) called."); - - // TODO (jarrodnorwell) - + *out_mask = 0; return ResultSuccess; }; diff --git a/src/core/launch_params.cpp b/src/core/launch_params.cpp index 25fdd45fdb..6c436d0346 100644 --- a/src/core/launch_params.cpp +++ b/src/core/launch_params.cpp @@ -39,6 +39,7 @@ LaunchParams ParseLaunchParams(Core::System& system, int argc, char *argv[], wch {"null-render", no_argument, 0, 'n'}, {"singlecore", no_argument, 0, 's'}, {"filter", no_argument, 0, 'x'}, + {"pack-nup", required_argument, 0, 'N'}, {0, 0, 0, 0}, // clang-format on }; @@ -62,6 +63,9 @@ LaunchParams ParseLaunchParams(Core::System& system, int argc, char *argv[], wch case 'g': p.filepath = std::string(optarg); break; + case 'N': + p.pack_nup_source = std::string(optarg); + break; case 'i': { p.input_profile = std::string(optarg); break; @@ -196,7 +200,9 @@ void ApplyLaunchParams(LaunchParams const& lp) noexcept { "-p, --program Pass following string as arguments to executable\n" "-u, --user Select a specific user profile from 0 to 7\n" "-d, --debug Run the GDB stub on a port from 1 to 65535\n" - "-v, --version Output version information and exit\n", + "-v, --version Output version information and exit\n" + "--pack-nup <dump> [<out.nup>]" + " Pack a loose dump with all its updates and DLC into a NUP file and exit\n", lp.argv0 ); } diff --git a/src/core/launch_params.h b/src/core/launch_params.h index 4da30fc1c2..8d503770d8 100644 --- a/src/core/launch_params.h +++ b/src/core/launch_params.h @@ -23,6 +23,8 @@ struct LaunchParams { std::optional<int> selected_user{}; std::optional<u16> override_gdb_port{}; std::optional<std::string> log_filter{}; + // --pack-nup <dump> [<out.nup>]: packs a loose dump instead of booting a game (eden-cli). + std::optional<std::string> pack_nup_source{}; u16 port = Network::DefaultRoomPort; bool use_multiplayer = false; bool fullscreen = false; diff --git a/src/core/loader/deconstructed_rom_directory.cpp b/src/core/loader/deconstructed_rom_directory.cpp index 94dd3395ba..463a8c8930 100644 --- a/src/core/loader/deconstructed_rom_directory.cpp +++ b/src/core/loader/deconstructed_rom_directory.cpp @@ -8,6 +8,8 @@ #include <cctype> #include <chrono> #include <cstring> +#include <map> +#include <memory> #include <mutex> #include <string_view> #include "common/logging.h" @@ -92,31 +94,24 @@ FileSys::VirtualFile FindLooseRomFS(const FileSys::VirtualDir& exefs_dir) { } if (const auto parent = exefs_dir->GetParentDirectory()) { if (const auto romfs_dir = parent->GetSubdirectory("romfs")) { - // Keep no ownership here: launch preparation hands its image to the real loader. + // Building it walks every file and keeps an object for each (title 1: 206k files, ~150 + // MiB), so loaders of the same directory share one build while any of them holds it. static std::mutex cache_mutex; - static std::string cached_path; - static std::weak_ptr<FileSys::VfsFile> cached_romfs; - const auto path = romfs_dir->GetFullPath(); - { - std::scoped_lock lock{cache_mutex}; - if (cached_path == path) { - if (auto cached = cached_romfs.lock()) { - return cached; - } - } + static std::map<std::string, std::weak_ptr<FileSys::VfsFile>> cache; + std::scoped_lock lk{cache_mutex}; + auto& cached = cache[romfs_dir->GetFullPath()]; + if (auto built = cached.lock()) { + return built; } + LOG_INFO(Loader, "Using RomFS directory next to the ExeFS"); const auto started = std::chrono::steady_clock::now(); - LOG_INFO(Loader, "Building loose RomFS directory image"); - auto image = FileSys::CreateRomFS(romfs_dir); + auto built = FileSys::CreateRomFS(romfs_dir); LOG_INFO(Loader, "Loose RomFS image ready: elapsed_ms={}", std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now() - started).count()); - { - std::scoped_lock lock{cache_mutex}; - cached_path = path; - cached_romfs = image; - } - return image; + std::chrono::steady_clock::now() - started) + .count()); + cached = built; + return built; } } return nullptr; @@ -258,6 +253,7 @@ AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory(FileSys name = control->GetApplicationName(); } + romfs_exefs_dir = file_dir; if (const auto title_dir = file_dir->GetParentDirectory()) { romfs_version = RomFSProductVersion(title_dir->GetSubdirectory("romfs")); } @@ -269,13 +265,17 @@ AppLoader_DeconstructedRomDirectory::AppLoader_DeconstructedRomDirectory( , dir(std::move(directory)) , override_update(override_update_) { + romfs_exefs_dir = dir; } -void AppLoader_DeconstructedRomDirectory::EnsureRomFS() { - if (!romfs_initialized) { - romfs = FindLooseRomFS(dir ? dir : file->GetContainingDirectory()); - romfs_initialized = true; +// Loaders made only for metadata (the game list, title lookups) never need the RomFS, and several +// of them were alive with a whole copy each. +const FileSys::VirtualFile& AppLoader_DeconstructedRomDirectory::LooseRomFS() { + if (!romfs_resolved) { + romfs_resolved = true; + romfs = FindLooseRomFS(romfs_exefs_dir); } + return romfs; } FileType AppLoader_DeconstructedRomDirectory::IdentifyType(const FileSys::VirtualFile& dir_file) { @@ -336,7 +336,7 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect // Loose-ExeFS loads bypass the NCA/NSP loaders, which are the ones that normally apply // LayeredFS to the RomFS; apply it here so mods in user/load/<titleid>/ work too. - if (romfs != nullptr) { + if (LooseRomFS() != nullptr) { const FileSys::PatchManager patch_manager(metadata.GetTitleID(), system.GetFileSystemController(), system.GetContentProvider()); @@ -446,8 +446,7 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect } ResultStatus AppLoader_DeconstructedRomDirectory::ReadRomFS(FileSys::VirtualFile& out_dir) { - EnsureRomFS(); - if (romfs == nullptr) { + if (LooseRomFS() == nullptr) { return ResultStatus::ErrorNoRomFS; } diff --git a/src/core/loader/deconstructed_rom_directory.h b/src/core/loader/deconstructed_rom_directory.h index 477638defe..db0120f4ad 100644 --- a/src/core/loader/deconstructed_rom_directory.h +++ b/src/core/loader/deconstructed_rom_directory.h @@ -58,11 +58,13 @@ public: ResultStatus ReadNSOModules(Modules& out_modules) override; private: - void EnsureRomFS(); + /// The loose RomFS next to the ExeFS, found (and built) on first use. + const FileSys::VirtualFile& LooseRomFS(); FileSys::ProgramMetadata metadata; FileSys::VirtualFile romfs; - bool romfs_initialized{}; + FileSys::VirtualDir romfs_exefs_dir; ///< Where LooseRomFS looks + bool romfs_resolved{}; FileSys::VirtualDir dir; std::vector<u8> icon_data; diff --git a/src/core/loader/loader.cpp b/src/core/loader/loader.cpp index 33a93573cf..839bb4d80b 100644 --- a/src/core/loader/loader.cpp +++ b/src/core/loader/loader.cpp @@ -29,6 +29,7 @@ #include "core/loader/nro.h" #include "core/loader/nso.h" #include "core/loader/nsp.h" +#include "core/loader/nup.h" #include "core/loader/xci.h" namespace Loader { @@ -130,6 +131,8 @@ FileType IdentifyFile(FileSys::VirtualFile file) { return *nsp_type; } else if (const auto xci_type = IdentifyFileLoader<AppLoader_XCI>(file)) { return *xci_type; + } else if (const auto nup_type = IdentifyFileLoader<AppLoader_NUP>(file)) { + return *nup_type; } else if (const auto nro_type = IdentifyFileLoader<AppLoader_NRO>(file)) { return *nro_type; } else if (const auto nca_type = IdentifyFileLoader<AppLoader_NCA>(file)) { @@ -150,7 +153,7 @@ FileType IdentifyFile(FileSys::VirtualFile file) { bool IsContainerType(FileType type) { return type == FileType::NSP || type == FileType::NSZ || type == FileType::XCI || - type == FileType::XCZ; + type == FileType::XCZ || type == FileType::NUP; } bool IsBootableGameContainer(FileSys::VirtualFile file, FileType type, u64 program_id, @@ -163,6 +166,24 @@ bool IsBootableGameContainer(FileSys::VirtualFile file, FileType type, u64 progr type = IdentifyFile(file); } + if (type == FileType::NUP) { + FileSys::NupContainer nup{file}; + if (nup.GetStatus() != ResultStatus::Success) { + return false; + } + // A base package boots normally; a standalone update (no base) carries + // a complete patched tree and boots on its own (SPEC section 4.1.2). + if (const auto base = nup.FindBasePackage(); + base.has_value() && nup.GetExeFSDir(*base) != nullptr) { + return true; + } + if (const auto update = nup.FindBestUpdatePackage(); + update.has_value() && nup.GetExeFSDir(*update) != nullptr) { + return true; + } + return false; + } + if (!IsContainerType(type)) { return false; } @@ -190,6 +211,8 @@ FileType GuessFromFilename(const std::string& name) { return FileType::XCZ; else if (extension == "nsp") return FileType::NSP; + else if (extension == "nup") + return FileType::NUP; else if (extension == "nsz") return FileType::NSZ; else if (extension == "kip") @@ -215,6 +238,8 @@ std::string GetFileTypeString(FileType type) { return "NSP"; case FileType::NSZ: return "NSZ"; + case FileType::NUP: + return "NUP"; case FileType::KIP: return "KIP"; case FileType::DeconstructedRomDirectory: @@ -227,7 +252,7 @@ std::string GetFileTypeString(FileType type) { return "unknown"; } -constexpr std::array<const char*, 68> RESULT_MESSAGES{ +constexpr std::array<const char*, 70> RESULT_MESSAGES{ "The operation completed successfully.", "The loader requested to load is already loaded.", "The operation is not implemented.", @@ -296,6 +321,8 @@ constexpr std::array<const char*, 68> RESULT_MESSAGES{ "The INI file contains more than the maximum allowable number of KIP files.", "Integrity verification could not be performed for this file.", "Integrity verification failed.", + "The NUP container is malformed or its TOC hash does not match.", + "The NUP container does not contain a loadable base package.", }; std::string GetResultStatusString(ResultStatus status) { @@ -380,6 +407,10 @@ static std::unique_ptr<AppLoader> GetFileLoader(Core::System& system, FileSys::V case FileType::KIP: return std::make_unique<AppLoader_KIP>(std::move(file)); + // NUP (NX Package) container format. + case FileType::NUP: + return std::make_unique<AppLoader_NUP>(std::move(file)); + // NX deconstructed ROM directory. case FileType::DeconstructedRomDirectory: return std::make_unique<AppLoader_DeconstructedRomDirectory>(std::move(file)); diff --git a/src/core/loader/loader.h b/src/core/loader/loader.h index 431aeace73..23be3597f5 100644 --- a/src/core/loader/loader.h +++ b/src/core/loader/loader.h @@ -47,6 +47,7 @@ enum class FileType { XCZ, NAX, KIP, + NUP, DeconstructedRomDirectory, }; @@ -157,6 +158,8 @@ enum class ResultStatus : u16 { ErrorINITooManyKIPs, ErrorIntegrityVerificationNotImplemented, ErrorIntegrityVerificationFailed, + ErrorNUPBadFormat, + ErrorNUPNoBasePackage, }; std::string GetResultStatusString(ResultStatus status); diff --git a/src/core/loader/nup.cpp b/src/core/loader/nup.cpp new file mode 100644 index 0000000000..7ddfb8947e --- /dev/null +++ b/src/core/loader/nup.cpp @@ -0,0 +1,280 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "core/loader/nup.h" + +#include <algorithm> +#include <cstring> + +#include "common/logging.h" +#include "common/string_util.h" +#include "core/core.h" +#include "core/file_sys/nup_container.h" +#include "core/file_sys/control_metadata.h" +#include "core/file_sys/patch_manager.h" +#include "core/file_sys/romfs.h" +#include "core/file_sys/romfs_factory.h" +#include "core/file_sys/vfs/vfs_vector.h" +#include "core/hle/kernel/k_process.h" +#include "core/hle/service/filesystem/filesystem.h" +#include "core/loader/deconstructed_rom_directory.h" + +namespace Loader { + +namespace { + +using FileSys::NupContainer; + +// NACP stores 16 uncompressed language entries; map BCP-47 tags onto them. +struct LanguageSlot { + std::string_view tag; + std::size_t index; +}; + +constexpr std::array<LanguageSlot, 18> LanguageSlots{{ + {"en-us", 0}, {"en-gb", 1}, {"ja", 2}, {"fr", 3}, {"de", 4}, + {"es-419", 5}, {"es", 6}, {"it", 7}, {"nl", 8}, {"fr-ca", 9}, + {"pt", 10}, {"ru", 11}, {"ko", 12}, {"zh-hant", 13}, {"zh-hans", 14}, + {"pt-br", 15}, {"pl", 16}, {"th", 17}, +}}; + +void CopyNameToEntry(FileSys::LanguageEntry& entry, std::string_view name) { + const std::size_t length = std::min(name.size(), entry.application_name.size() - 1); + std::memcpy(entry.application_name.data(), name.data(), length); + entry.application_name[length] = '\0'; +} + +std::unique_ptr<FileSys::NACP> BuildNacp(const NupContainer::PackageInfo& base, + const std::string& version) { + if (base.title_name.empty() && base.localized_titles.empty() && version.empty()) { + return nullptr; + } + auto nacp = std::make_unique<FileSys::NACP>(); + nacp->raw.supported_languages = static_cast<FileSys::SupportedLanguage>(0x3FFFF); + + for (const auto& [lang, name] : base.localized_titles) { + for (const auto& slot : LanguageSlots) { + // The uncompressed RawNACP only stores 16 entries; Polish/Thai exist + // only in the compressed format, so they fall back to the default. + if (slot.index < 16 && slot.tag == lang) { + CopyNameToEntry(nacp->raw.language_entries.language_entries[slot.index], name); + } + } + } + if (!base.title_name.empty()) { + CopyNameToEntry(nacp->raw.language_entries.language_entries[0], base.title_name); + } + if (!version.empty()) { + const std::size_t length = std::min(version.size(), nacp->raw.version_string.size() - 1); + std::memcpy(nacp->raw.version_string.data(), version.data(), length); + nacp->raw.version_string[length] = '\0'; + } + nacp->raw.save_data_owner_id = base.title_id; + + nacp->language_entries.resize(16); + std::memcpy(nacp->language_entries.data(), nacp->raw.language_entries.language_entries.data(), + sizeof(nacp->raw.language_entries.language_entries)); + return nacp; +} + +std::string DecodeVersion(u32 version) { + if (version == 0) { + return {}; + } + return fmt::format("{}.{}.{}", version >> 16, (version >> 8) & 0xFF, version & 0xFF); +} + +} // namespace + +AppLoader_NUP::AppLoader_NUP(FileSys::VirtualFile file_) + : AppLoader(std::move(file_)), nup(std::make_unique<NupContainer>(file)) { + if (nup->GetStatus() != ResultStatus::Success) { + return; + } + + const auto base_index = nup->FindBasePackage(); + const auto update_index = nup->FindBestUpdatePackage(); + + // The program comes from the base package; a standalone update container + // (SPEC §4.1.2) carries a complete patched tree and boots on its own. + std::optional<std::size_t> program_index; + std::optional<std::size_t> exefs_index; + if (base_index && nup->GetExeFSDir(*base_index) != nullptr) { + program_index = base_index; + exefs_index = update_index && nup->GetExeFSDir(*update_index) != nullptr + ? update_index + : base_index; + } else if (update_index && nup->GetExeFSDir(*update_index) != nullptr) { + LOG_INFO(Loader, + "NUP container is a standalone update; booting the update program directly"); + program_index = update_index; + exefs_index = update_index; + } + if (!program_index || !exefs_index) { + LOG_ERROR(Loader, "NUP container has no package with a usable ExeFS"); + return; + } + + const auto& program = nup->GetPackages()[*program_index]; + base_title_id = program.title_id; + has_program = true; + + const auto exefs = nup->GetExeFSDir(*exefs_index); + if (exefs == nullptr || exefs->GetFile("main") == nullptr || + exefs->GetFile("main.npdm") == nullptr) { + LOG_ERROR(Loader, "NUP package has no usable ExeFS"); + return; + } + secondary_loader = std::make_unique<AppLoader_DeconstructedRomDirectory>(exefs, true); + + if (update_index) { + const auto& update = nup->GetPackages()[*update_index]; + display_version = update.display_version.empty() ? DecodeVersion(update.version) + : update.display_version; + } else { + display_version = program.display_version.empty() ? DecodeVersion(program.version) + : program.display_version; + } + nacp_file = BuildNacp(program, display_version); + if (nacp_file != nullptr && !program.icon.empty()) { + icon_file = std::make_shared<FileSys::VectorVfsFile>(program.icon, "icon.dat"); + } else if (nacp_file != nullptr) { + for (const auto& [lang, icon] : program.localized_icons) { + if (lang == "en-us") { + icon_file = std::make_shared<FileSys::VectorVfsFile>(icon, "icon_" + lang + ".dat"); + break; + } + } + if (icon_file == nullptr && !program.localized_icons.empty()) { + icon_file = std::make_shared<FileSys::VectorVfsFile>( + program.localized_icons.front().second, "icon.dat"); + } + } + + const auto addon_count = std::count_if(nup->GetPackages().begin(), nup->GetPackages().end(), + [](const auto& package) { + return package.kind == + NupContainer::PackageKind::Addon; + }); + if (addon_count > 0) { + LOG_WARNING(Loader, + "NUP container has {} add-on content package(s); AoC mounting is not " + "supported yet", + addon_count); + } +} + +AppLoader_NUP::~AppLoader_NUP() = default; + +FileType AppLoader_NUP::IdentifyType(const FileSys::VirtualFile& nup_file) { + return FileSys::NupContainer::IsNupFile(nup_file) ? FileType::NUP : FileType::Error; +} + +AppLoader::LoadResult AppLoader_NUP::Load(Kernel::KProcess& process, Core::System& system) { + if (is_loaded) { + return {ResultStatus::ErrorAlreadyLoaded, {}}; + } + if (nup->GetStatus() != ResultStatus::Success) { + return {nup->GetStatus(), {}}; + } + if (!has_program || secondary_loader == nullptr) { + return {ResultStatus::ErrorNUPNoBasePackage, {}}; + } + + const auto result = secondary_loader->Load(process, system); + if (result.first != ResultStatus::Success) { + return result; + } + + system.GetFileSystemController().RegisterProcess( + process.GetProcessId(), base_title_id, + std::make_shared<FileSys::RomFSFactory>(*this, system.GetContentProvider(), + system.GetFileSystemController())); + + is_loaded = true; + return result; +} + +ResultStatus AppLoader_NUP::VerifyIntegrity( + std::function<bool(size_t, size_t)> progress_callback) { + if (nup->GetStatus() != ResultStatus::Success) { + return nup->GetStatus(); + } + return nup->VerifyIntegrity(std::move(progress_callback)); +} + +ResultStatus AppLoader_NUP::ReadRomFS(FileSys::VirtualFile& out_file) { + const auto base_index = nup->FindBasePackage(); + const auto base_romfs = base_index ? nup->GetRomFSDir(*base_index) : FileSys::VirtualDir{}; + const auto update_index = nup->FindBestUpdatePackage(); + const auto update_romfs = + update_index ? nup->GetRomFSDir(*update_index) : FileSys::VirtualDir{}; + // A standalone update carries the complete merged tree; a base+update pair + // overlays the update onto the base (SPEC §4.1.4). + const auto merged = FileSys::NupContainer::MergeShadowDirs(base_romfs, update_romfs); + if (merged == nullptr) { + return ResultStatus::ErrorNoRomFS; + } + out_file = FileSys::CreateRomFS(merged); + if (out_file == nullptr) { + return ResultStatus::ErrorNoRomFS; + } + return ResultStatus::Success; +} + +bool AppLoader_NUP::IsRomFSUpdatable() const { + // The update is already merged into the exposed RomFS. + return false; +} + +ResultStatus AppLoader_NUP::ReadProgramId(u64& out_program_id) { + if (!has_program) { + return ResultStatus::ErrorNotInitialized; + } + out_program_id = base_title_id; + return ResultStatus::Success; +} + +ResultStatus AppLoader_NUP::ReadProgramIds(std::vector<u64>& out_program_ids) { + if (!has_program) { + return ResultStatus::ErrorNotInitialized; + } + out_program_ids.push_back(base_title_id); + return ResultStatus::Success; +} + +ResultStatus AppLoader_NUP::ReadIcon(std::vector<u8>& buffer) { + if (icon_file == nullptr) { + return ResultStatus::ErrorNoIcon; + } + buffer = icon_file->ReadAllBytes(); + return ResultStatus::Success; +} + +ResultStatus AppLoader_NUP::ReadTitle(std::string& title) { + if (nacp_file == nullptr) { + return ResultStatus::ErrorNoControl; + } + title = nacp_file->GetApplicationName(); + if (title.empty()) { + return ResultStatus::ErrorNoControl; + } + return ResultStatus::Success; +} + +ResultStatus AppLoader_NUP::ReadControlData(FileSys::NACP& control) { + if (nacp_file == nullptr) { + return ResultStatus::ErrorNoControl; + } + control = *nacp_file; + return ResultStatus::Success; +} + +ResultStatus AppLoader_NUP::ReadNSOModules(Modules& modules) { + if (secondary_loader == nullptr) { + return ResultStatus::ErrorNotInitialized; + } + return secondary_loader->ReadNSOModules(modules); +} + +} // namespace Loader diff --git a/src/core/loader/nup.h b/src/core/loader/nup.h new file mode 100644 index 0000000000..1f9dd5922b --- /dev/null +++ b/src/core/loader/nup.h @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <memory> +#include <optional> +#include <string> + +#include "core/file_sys/nup_container.h" +#include "core/loader/loader.h" + +namespace FileSys { +class NACP; +class NupContainer; +} // namespace FileSys + +namespace Loader { + +/// Loads a NUP (NX Package) container: loose ExeFS/RomFS trees for a base +/// program plus its updates in one file. The newest update package is layered +/// over the base; add-on content packages are recognized but not yet mounted. +class AppLoader_NUP final : public AppLoader { +public: + explicit AppLoader_NUP(FileSys::VirtualFile file_); + ~AppLoader_NUP() override; + + /** + * Identifies whether or not the given file is a NUP container. + * + * @param nup_file The file to identify. + * + * @return FileType::NUP, or FileType::Error if the file is not a NUP. + */ + static FileType IdentifyType(const FileSys::VirtualFile& nup_file); + + FileType GetFileType() const override { + return FileType::NUP; + } + + LoadResult Load(Kernel::KProcess& process, Core::System& system) override; + + ResultStatus VerifyIntegrity(std::function<bool(size_t, size_t)> progress_callback) override; + + ResultStatus ReadRomFS(FileSys::VirtualFile& out_file) override; + bool IsRomFSUpdatable() const override; + ResultStatus ReadProgramId(u64& out_program_id) override; + ResultStatus ReadProgramIds(std::vector<u64>& out_program_ids) override; + ResultStatus ReadIcon(std::vector<u8>& buffer) override; + ResultStatus ReadTitle(std::string& title) override; + ResultStatus ReadControlData(FileSys::NACP& control) override; + ResultStatus ReadNSOModules(Modules& modules) override; + +private: + std::unique_ptr<FileSys::NupContainer> nup; + std::unique_ptr<AppLoader> secondary_loader; + + std::unique_ptr<FileSys::NACP> nacp_file; + FileSys::VirtualFile icon_file; + u64 base_title_id{}; + std::string display_version; + bool has_program{}; +}; + +} // namespace Loader diff --git a/src/dynarmic/src/dynarmic/backend/block_range_information.cpp b/src/dynarmic/src/dynarmic/backend/block_range_information.cpp index 1a56b7adb6..068e86c2bc 100644 --- a/src/dynarmic/src/dynarmic/backend/block_range_information.cpp +++ b/src/dynarmic/src/dynarmic/backend/block_range_information.cpp @@ -8,7 +8,8 @@ #include "dynarmic/backend/block_range_information.h" -#include <boost/icl/interval_map.hpp> +#include <algorithm> + #include <boost/icl/interval_set.hpp> #include "common/common_types.h" #include "common/container/unordered_set.h" @@ -17,24 +18,28 @@ namespace Dynarmic::Backend { template<typename P> void BlockRangeInformation<P>::AddRange(boost::icl::discrete_interval<P> range, IR::LocationDescriptor location) { - block_ranges.add(std::make_pair(range, ::Common::unordered_set<IR::LocationDescriptor>{location})); + if (!boost::icl::is_empty(range)) { + ranges.push_back({boost::icl::first(range), boost::icl::last(range), location}); + } } template<typename P> void BlockRangeInformation<P>::ClearCache() { - block_ranges.clear(); + ranges.clear(); } template<typename P> -::Common::unordered_set<IR::LocationDescriptor> BlockRangeInformation<P>::InvalidateRanges(const boost::icl::interval_set<P>& ranges) { +::Common::unordered_set<IR::LocationDescriptor> BlockRangeInformation<P>::InvalidateRanges(const boost::icl::interval_set<P>& invalidated) { ::Common::unordered_set<IR::LocationDescriptor> erase_locations; - for (auto invalidate_interval : ranges) { - auto pair = block_ranges.equal_range(invalidate_interval); - for (auto it = pair.first; it != pair.second; ++it) - for (const auto& descriptor : it->second) - erase_locations.insert(descriptor); - } - // TODO: EFFICIENCY: Remove ranges that are to be erased. + std::erase_if(ranges, [&](const Range& range) { + for (const auto& interval : invalidated) { + if (range.first <= boost::icl::last(interval) && boost::icl::first(interval) <= range.last) { + erase_locations.insert(range.location); + return true; + } + } + return false; + }); return erase_locations; } diff --git a/src/dynarmic/src/dynarmic/backend/block_range_information.h b/src/dynarmic/src/dynarmic/backend/block_range_information.h index 8f7daa9d9c..f894240603 100644 --- a/src/dynarmic/src/dynarmic/backend/block_range_information.h +++ b/src/dynarmic/src/dynarmic/backend/block_range_information.h @@ -8,11 +8,9 @@ #pragma once -#include <set> +#include <vector> -#include <boost/icl/interval_map.hpp> #include <boost/icl/interval_set.hpp> -#include "common/container/unordered_map.h" #include "common/container/unordered_set.h" #include "dynarmic/ir/location_descriptor.h" @@ -24,8 +22,18 @@ class BlockRangeInformation { public: void AddRange(boost::icl::discrete_interval<P> range, IR::LocationDescriptor location); void ClearCache(); - ::Common::unordered_set<IR::LocationDescriptor> InvalidateRanges(const boost::icl::interval_set<P>& ranges); - boost::icl::interval_map<P, ::Common::unordered_set<IR::LocationDescriptor>> block_ranges; + ::Common::unordered_set<IR::LocationDescriptor> InvalidateRanges(const boost::icl::interval_set<P>& invalidated); + +private: + /// The guest code [first, last] a block was translated from. Code is invalidated rarely (when + /// it is unmapped or reprotected), so a list scanned then costs least: an interval map of + /// block sets held ~300 bytes per block. + struct Range { + P first; + P last; + IR::LocationDescriptor location; + }; + std::vector<Range> ranges; }; } // namespace Dynarmic::Backend diff --git a/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.cpp b/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.cpp index 5c6a90e15a..e0e3c463d1 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.cpp +++ b/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.cpp @@ -1168,7 +1168,7 @@ bool EmitTerminalImpl(A32EmitX64& e, IR::Term::LinkBlock terminal, IR::LocationD } else { if (e.conf.enable_cycle_counting) { e.code.cmp(qword[rsp + ABI_SHADOW_SPACE + offsetof(StackLayout, cycles_remaining)], 0); - e.patch_information[terminal.next].jg.push_back(e.code.getCurr()); + e.AddPatchLocation(terminal.next, EmitX64::PatchKind::Jg); if (const auto next_bb = e.GetBasicBlock(terminal.next)) { e.EmitPatchJg(terminal.next, next_bb->entrypoint); } else { @@ -1176,7 +1176,7 @@ bool EmitTerminalImpl(A32EmitX64& e, IR::Term::LinkBlock terminal, IR::LocationD } } else { e.code.cmp(dword[e.code.ABI_JIT_PTR + offsetof(A32JitState, halt_reason)], 0); - e.patch_information[terminal.next].jz.push_back(e.code.getCurr()); + e.AddPatchLocation(terminal.next, EmitX64::PatchKind::Jz); if (const auto next_bb = e.GetBasicBlock(terminal.next)) { e.EmitPatchJz(terminal.next, next_bb->entrypoint); } else { @@ -1196,7 +1196,7 @@ bool EmitTerminalImpl(A32EmitX64& e, IR::Term::LinkBlockFast terminal, IR::Locat e.code.mov(MJitStateReg(A32::Reg::PC), A32::LocationDescriptor{terminal.next}.PC()); e.code.ReturnFromRunCode(); } else { - e.patch_information[terminal.next].jmp.push_back(e.code.getCurr()); + e.AddPatchLocation(terminal.next, EmitX64::PatchKind::Jmp); if (const auto next_bb = e.GetBasicBlock(terminal.next)) { e.EmitPatchJmp(terminal.next, next_bb->entrypoint); } else { diff --git a/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.h b/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.h index a106bb6482..eb36bffd88 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.h +++ b/src/dynarmic/src/dynarmic/backend/x64/a32_emit_x64.h @@ -87,13 +87,17 @@ public: // Fastmem information using DoNotFastmemMarker = std::tuple<IR::LocationDescriptor, unsigned>; + /// Where a fastmem access that faults goes: code addresses are offsets in the code space. struct FastmemPatchInfo { - u64 resume_rip; - u64 callback; - DoNotFastmemMarker marker; - bool recompile; + u32 rip; + u32 resume_rip; + u32 callback; + u32 inst_name : 31; + u32 recompile : 1; + IR::LocationDescriptor location; }; std::optional<DoNotFastmemMarker> ShouldFastmem(A32EmitContext& ctx, IR::Inst* inst) const; + void AddFastmemPatch(CodePtr location, CodePtr resume_location, void (*callback)(), const DoNotFastmemMarker& marker, bool recompile); FakeCall FastmemCallback(u64 rip); // Memory access helpers @@ -127,7 +131,7 @@ public: RegAlloc reg_alloc; //reusable reg alloc BlockRangeInformation<u32> block_ranges; std::array<FastDispatchEntry, fast_dispatch_table_size> fast_dispatch_table; - ::Common::unordered_map<u64, FastmemPatchInfo> fastmem_patch_info; + std::vector<FastmemPatchInfo> fastmem_patch_info; // sorted by rip ::Common::unordered_map<std::tuple<bool, size_t, int, int>, void (*)()> read_fallbacks; ::Common::unordered_map<std::tuple<bool, size_t, int, int>, void (*)()> write_fallbacks; ::Common::unordered_map<std::tuple<bool, size_t, int, int>, void (*)()> exclusive_write_fallbacks; diff --git a/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.cpp b/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.cpp index 71e5e62cd9..2307226744 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.cpp +++ b/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.cpp @@ -628,7 +628,7 @@ bool EmitTerminalImpl(A64EmitX64& e, IR::Term::LinkBlock terminal, IR::LocationD if (e.conf.HasOptimization(OptimizationFlag::BlockLinking) && !is_single_step) { if (e.conf.enable_cycle_counting) { e.code.cmp(qword[rsp + ABI_SHADOW_SPACE + offsetof(StackLayout, cycles_remaining)], 0); - e.patch_information[terminal.next].jg.push_back(e.code.getCurr()); + e.AddPatchLocation(terminal.next, EmitX64::PatchKind::Jg); if (const auto next_bb = e.GetBasicBlock(terminal.next)) { e.EmitPatchJg(terminal.next, next_bb->entrypoint); } else { @@ -636,7 +636,7 @@ bool EmitTerminalImpl(A64EmitX64& e, IR::Term::LinkBlock terminal, IR::LocationD } } else { e.code.cmp(dword[e.code.ABI_JIT_PTR + offsetof(A64JitState, halt_reason)], 0); - e.patch_information[terminal.next].jz.push_back(e.code.getCurr()); + e.AddPatchLocation(terminal.next, EmitX64::PatchKind::Jz); if (const auto next_bb = e.GetBasicBlock(terminal.next)) { e.EmitPatchJz(terminal.next, next_bb->entrypoint); } else { @@ -656,7 +656,7 @@ bool EmitTerminalImpl(A64EmitX64& e, IR::Term::LinkBlock terminal, IR::LocationD bool EmitTerminalImpl(A64EmitX64& e, IR::Term::LinkBlockFast terminal, IR::LocationDescriptor, bool is_single_step) { if (e.conf.HasOptimization(OptimizationFlag::BlockLinking) && !is_single_step) { - e.patch_information[terminal.next].jmp.push_back(e.code.getCurr()); + e.AddPatchLocation(terminal.next, EmitX64::PatchKind::Jmp); if (auto next_bb = e.GetBasicBlock(terminal.next)) { e.EmitPatchJmp(terminal.next, next_bb->entrypoint); } else { diff --git a/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.h b/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.h index 1310b4ad34..a9f2d9b650 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.h +++ b/src/dynarmic/src/dynarmic/backend/x64/a64_emit_x64.h @@ -83,13 +83,17 @@ public: // Fastmem information using DoNotFastmemMarker = std::tuple<IR::LocationDescriptor, unsigned>; + /// Where a fastmem access that faults goes: code addresses are offsets in the code space. struct FastmemPatchInfo { - u64 resume_rip; - u64 callback; - DoNotFastmemMarker marker; - bool recompile; + u32 rip; + u32 resume_rip; + u32 callback; + u32 inst_name : 31; + u32 recompile : 1; + IR::LocationDescriptor location; }; std::optional<DoNotFastmemMarker> ShouldFastmem(A64EmitContext& ctx, IR::Inst* inst) const; + void AddFastmemPatch(CodePtr location, CodePtr resume_location, void (*callback)(), const DoNotFastmemMarker& marker, bool recompile); FakeCall FastmemCallback(u64 rip); // Memory access helpers @@ -123,7 +127,7 @@ public: RegAlloc reg_alloc; //reusable reg alloc BlockRangeInformation<u64> block_ranges; std::array<FastDispatchEntry, fast_dispatch_table_size> fast_dispatch_table; - ::Common::unordered_map<u64, FastmemPatchInfo> fastmem_patch_info; + std::vector<FastmemPatchInfo> fastmem_patch_info; // sorted by rip ::Common::unordered_map<std::tuple<bool, size_t, int, int>, void (*)()> read_fallbacks; ::Common::unordered_map<std::tuple<bool, size_t, int, int>, void (*)()> write_fallbacks; ::Common::unordered_map<std::tuple<bool, size_t, int, int>, void (*)()> exclusive_write_fallbacks; diff --git a/src/dynarmic/src/dynarmic/backend/x64/block_of_code.cpp b/src/dynarmic/src/dynarmic/backend/x64/block_of_code.cpp index 384d6203a4..57001d4ef0 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/block_of_code.cpp +++ b/src/dynarmic/src/dynarmic/backend/x64/block_of_code.cpp @@ -60,6 +60,34 @@ namespace { constexpr size_t CONSTANT_POOL_SIZE = 2 * 1024 * 1024; constexpr size_t PRELUDE_COMMIT_SIZE = 16 * 1024 * 1024; +#ifdef _WIN32 +constexpr size_t COMMIT_STEP = 4 * 1024 * 1024; + +// Reserves the code space; EnsureMemoryCommitted commits it as code is emitted. Xbyak's default +// allocator commits all of it up front: 512 MiB per core, 3 GiB of commit charge for six cores +// of which a game fills a fraction. +struct ReservingAllocator final : Xbyak::Allocator { + uint8_t* alloc(size_t size) override { + return static_cast<uint8_t*>(VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_NOACCESS)); + } + void free(uint8_t* p) override { + VirtualFree(p, 0, MEM_RELEASE); + } + bool useProtect() const override { + return false; + } +}; + +Xbyak::Allocator* CodeAllocator() { + // Never destroyed: code blocks may outlive static destruction. + static auto* const allocator = new ReservingAllocator; + return allocator; +} +#else +Xbyak::Allocator* CodeAllocator() { + return nullptr; +} +#endif #ifdef DYNARMIC_ENABLE_NO_EXECUTE_SUPPORT void ProtectMemory(const void* base, size_t size, bool is_executable) { @@ -149,7 +177,7 @@ BlockOfCode::BlockOfCode(RunCodeCallbacks cb, JitStateInfo jsi, size_t total_cod #else , nullptr //Allow RWE #endif - , nullptr) + , CodeAllocator()) , constant_pool(*this, CONSTANT_POOL_SIZE) , jsi(jsi) , cb(std::move(cb)) @@ -206,7 +234,9 @@ size_t BlockOfCode::SpaceRemaining() const { void BlockOfCode::EnsureMemoryCommitted([[maybe_unused]] size_t codesize) { #ifdef _WIN32 if (committed_size < size_ + codesize) { - committed_size = std::min<size_t>(maxSize_, committed_size + codesize); + // The code space is only reserved (CodeAllocator), so everything up to the end of the next + // emission has to be committed; in steps, as each VirtualAlloc call is a kernel call. + committed_size = std::min<size_t>(maxSize_, (std::max)(committed_size + COMMIT_STEP, size_ + codesize)); # ifdef DYNARMIC_ENABLE_NO_EXECUTE_SUPPORT VirtualAlloc(top_, committed_size, MEM_COMMIT, PAGE_READWRITE); # else diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64.cpp b/src/dynarmic/src/dynarmic/backend/x64/emit_x64.cpp index b9df3987ca..84b9888bf5 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64.cpp +++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64.cpp @@ -102,7 +102,7 @@ void EmitX64::PushRSBHelper(Xbyak::Reg64 loc_desc_reg, Xbyak::Reg64 index_reg, I code.mov(index_reg.cvt32(), dword[code.ABI_JIT_PTR + code.GetJitStateInfo().offsetof_rsb_ptr]); code.mov(loc_desc_reg, target.Value()); - patch_information[target].mov_rcx.push_back(code.getCurr()); + AddPatchLocation(target, PatchKind::MovRcx); EmitPatchMovRcx(target_code_ptr); code.mov(qword[code.ABI_JIT_PTR + index_reg * 8 + code.GetJitStateInfo().offsetof_rsb_location_descriptors], loc_desc_reg); code.mov(qword[code.ABI_JIT_PTR + index_reg * 8 + code.GetJitStateInfo().offsetof_rsb_codeptrs], rcx); @@ -357,37 +357,61 @@ EmitX64::BlockDescriptor EmitX64::RegisterBlock(const IR::LocationDescriptor& de return block_desc; } -void EmitX64::Patch(const IR::LocationDescriptor& target_desc, CodePtr target_code_ptr) { - const CodePtr save_code_ptr = code.getCurr(); - const PatchInformation& patch_info = patch_information[target_desc]; - - for (CodePtr location : patch_info.jg) { - code.SetCodePtr(location); - EmitPatchJg(target_desc, target_code_ptr); +void EmitX64::AddPatchLocation(const IR::LocationDescriptor& target_desc, PatchKind kind) { + const size_t offset = code.getCurr<const u8*>() - static_cast<const u8*>(code.getCode()); + ASSERT(offset > 0 && offset < (size_t(1) << 30)); + const u32 entry = u32(offset << 2) | u32(kind); + PatchInformation& patch_info = patch_information[target_desc]; + for (u32& slot : patch_info.first) { + if (slot == 0) { + slot = entry; + return; + } } - - for (CodePtr location : patch_info.jz) { - code.SetCodePtr(location); - EmitPatchJz(target_desc, target_code_ptr); + if (!patch_info.rest) { + patch_info.rest = std::make_unique<std::vector<u32>>(); } + patch_info.rest->push_back(entry); +} - for (CodePtr location : patch_info.jmp) { - code.SetCodePtr(location); - EmitPatchJmp(target_desc, target_code_ptr); +void EmitX64::Patch(const IR::LocationDescriptor& target_desc, CodePtr target_code_ptr) { + const auto it = patch_information.find(target_desc); + if (it == patch_information.end()) { + return; } - - for (CodePtr location : patch_info.mov_rcx) { - code.SetCodePtr(location); - EmitPatchMovRcx(target_code_ptr); + const CodePtr save_code_ptr = code.getCurr(); + const auto patch = [&](u32 entry) { + code.SetCodePtr(static_cast<const u8*>(code.getCode()) + (entry >> 2)); + switch (PatchKind(entry & 3)) { + case PatchKind::Jg: + EmitPatchJg(target_desc, target_code_ptr); + break; + case PatchKind::Jz: + EmitPatchJz(target_desc, target_code_ptr); + break; + case PatchKind::Jmp: + EmitPatchJmp(target_desc, target_code_ptr); + break; + case PatchKind::MovRcx: + EmitPatchMovRcx(target_code_ptr); + break; + } + }; + for (const u32 entry : it->second.first) { + if (entry != 0) { + patch(entry); + } + } + if (it->second.rest) { + for (const u32 entry : *it->second.rest) { + patch(entry); + } } - code.SetCodePtr(save_code_ptr); } void EmitX64::Unpatch(const IR::LocationDescriptor& target_desc) { - if (patch_information.count(target_desc)) { - Patch(target_desc, nullptr); - } + Patch(target_desc, nullptr); } void EmitX64::ClearCache() { diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64.h b/src/dynarmic/src/dynarmic/backend/x64/emit_x64.h index 5e540e5f70..327cac7668 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64.h +++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64.h @@ -116,12 +116,16 @@ public: virtual bool EmitTerminal(IR::Term::Terminal const& terminal, IR::LocationDescriptor initial_location, bool is_single_step) noexcept = 0; // Patching + enum class PatchKind : u32 { Jg, Jz, Jmp, MovRcx }; + /// Places in emitted code that jump to a block (or load its entrypoint), patched when it is + /// compiled or invalidated: each the place's offset in the code space times four plus its + /// PatchKind, 0 when unused. Most blocks are linked from one or two places. struct PatchInformation { - boost::container::small_vector<CodePtr, 4> jg; //4*8=32 - boost::container::small_vector<CodePtr, 4> jz; //4*8=32 - boost::container::small_vector<CodePtr, 4> jmp; //4*8=32 - boost::container::small_vector<CodePtr, 4> mov_rcx; //4*8=32 + std::array<u32, 2> first{}; + std::unique_ptr<std::vector<u32>> rest; }; + /// Records that the code about to be emitted at the current position is a patch of `kind`. + void AddPatchLocation(const IR::LocationDescriptor& target_desc, PatchKind kind); void Patch(const IR::LocationDescriptor& target_desc, CodePtr target_code_ptr); virtual void Unpatch(const IR::LocationDescriptor& target_desc); virtual void EmitPatchJg(const IR::LocationDescriptor& target_desc, CodePtr target_code_ptr = nullptr) = 0; diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.cpp.inc b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.cpp.inc index 8bb8be3d5b..689e507abe 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.cpp.inc +++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.cpp.inc @@ -30,15 +30,48 @@ std::optional<AxxEmitX64::DoNotFastmemMarker> AxxEmitX64::ShouldFastmem(AxxEmitC return marker; } +void AxxEmitX64::AddFastmemPatch(CodePtr location, CodePtr resume_location, void (*callback)(), const DoNotFastmemMarker& marker, bool recompile) { + const u64 base = std::bit_cast<u64>(code.getCode()); + const auto offset = [base](u64 address) { + ASSERT(address >= base && address - base <= UINT32_MAX); + return u32(address - base); + }; + FastmemPatchInfo info{ + .rip = offset(std::bit_cast<u64>(location)), + .resume_rip = offset(std::bit_cast<u64>(resume_location)), + .callback = offset(std::bit_cast<u64>(callback)), + .inst_name = 0, + .recompile = 0, + .location = std::get<0>(marker), + }; + info.inst_name = std::get<1>(marker); + info.recompile = recompile ? 1 : 0; + // Code is emitted in address order; only a block's deferred fallbacks can come after a later + // access of the same block. + auto it = fastmem_patch_info.end(); + if (!fastmem_patch_info.empty() && fastmem_patch_info.back().rip >= info.rip) { + it = std::lower_bound(fastmem_patch_info.begin(), fastmem_patch_info.end(), info.rip, + [](const FastmemPatchInfo& entry, u32 value) { return entry.rip < value; }); + if (it != fastmem_patch_info.end() && it->rip == info.rip) { + return; + } + } + fastmem_patch_info.insert(it, info); +} + FakeCall AxxEmitX64::FastmemCallback(u64 rip_) { - const auto iter = fastmem_patch_info.find(rip_); - if (iter != fastmem_patch_info.end()) { + const u64 base = std::bit_cast<u64>(code.getCode()); + const auto iter = rip_ >= base && rip_ - base <= UINT32_MAX + ? std::lower_bound(fastmem_patch_info.begin(), fastmem_patch_info.end(), u32(rip_ - base), + [](const FastmemPatchInfo& entry, u32 value) { return entry.rip < value; }) + : fastmem_patch_info.end(); + if (iter != fastmem_patch_info.end() && iter->rip == u32(rip_ - base)) { FakeCall result{ - .call_rip = iter->second.callback, - .ret_rip = iter->second.resume_rip, + .call_rip = base + iter->callback, + .ret_rip = base + iter->resume_rip, }; - if (iter->second.recompile) { - const auto marker = iter->second.marker; + if (iter->recompile) { + const DoNotFastmemMarker marker{iter->location, static_cast<unsigned>(iter->inst_name)}; do_not_fastmem.insert(marker); InvalidateBasicBlocks({std::get<0>(marker)}); } @@ -99,14 +132,7 @@ void AxxEmitX64::EmitMemoryRead(AxxEmitContext& ctx, IR::Inst* inst) { code.L(*abort); code.call(wrapped_fn); - fastmem_patch_info.emplace( - std::bit_cast<u64>(location), - FastmemPatchInfo{ - std::bit_cast<u64>(code.getCurr()), - std::bit_cast<u64>(wrapped_fn), - *fastmem_marker, - conf.recompile_on_fastmem_failure, - }); + AddFastmemPatch(location, code.getCurr(), wrapped_fn, *fastmem_marker, conf.recompile_on_fastmem_failure); EmitCheckMemoryAbort(ctx, inst, end); code.jmp(*end, code.T_NEAR); @@ -186,14 +212,7 @@ void AxxEmitX64::EmitMemoryWrite(AxxEmitContext& ctx, IR::Inst* inst) { code.L(*abort); code.call(wrapped_fn); - fastmem_patch_info.emplace( - std::bit_cast<u64>(location), - FastmemPatchInfo{ - std::bit_cast<u64>(code.getCurr()), - std::bit_cast<u64>(wrapped_fn), - *fastmem_marker, - conf.recompile_on_fastmem_failure, - }); + AddFastmemPatch(location, code.getCurr(), wrapped_fn, *fastmem_marker, conf.recompile_on_fastmem_failure); EmitCheckMemoryAbort(ctx, inst, end); code.jmp(*end, code.T_NEAR); @@ -357,14 +376,7 @@ void AxxEmitX64::EmitExclusiveReadMemoryInline(AxxEmitContext& ctx, IR::Inst* in const auto location = EmitReadMemoryMov<bitsize>(code, value_idx, src_ptr, ordered); - fastmem_patch_info.emplace( - std::bit_cast<u64>(location), - FastmemPatchInfo{ - std::bit_cast<u64>(code.getCurr()), - std::bit_cast<u64>(wrapped_fn), - *fastmem_marker, - conf.recompile_on_exclusive_fastmem_failure, - }); + AddFastmemPatch(location, code.getCurr(), wrapped_fn, *fastmem_marker, conf.recompile_on_exclusive_fastmem_failure); code.L(*end); @@ -495,14 +507,7 @@ void AxxEmitX64::EmitExclusiveWriteMemoryInline(AxxEmitContext& ctx, IR::Inst* i code.L(*abort); code.call(wrapped_fn); - fastmem_patch_info.emplace( - std::bit_cast<u64>(location), - FastmemPatchInfo{ - std::bit_cast<u64>(code.getCurr()), - std::bit_cast<u64>(wrapped_fn), - *fastmem_marker, - conf.recompile_on_exclusive_fastmem_failure, - }); + AddFastmemPatch(location, code.getCurr(), wrapped_fn, *fastmem_marker, conf.recompile_on_exclusive_fastmem_failure); code.xor_(status.cvt32(), status.cvt32()); //dep-break code.test(code.al, code.al); diff --git a/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.cpp b/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.cpp index 5a828839c1..4f3d253441 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.cpp +++ b/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.cpp @@ -465,22 +465,24 @@ std::optional<HostLoc> RegAlloc::ValueLocation(const IR::Inst* value) const noex return std::nullopt; } +// The checks that scan every location for def_inst are debug-only: per IR instruction they cost as +// much as the rest of register allocation. void RegAlloc::DefineValueImpl(BlockOfCode& code, IR::Inst* def_inst, HostLoc host_loc) noexcept { - ASSERT(!ValueLocation(def_inst) && "def_inst has already been defined"); + DEBUG_ASSERT(!ValueLocation(def_inst) && "def_inst has already been defined"); LocInfo(host_loc).AddValue(host_loc, def_inst); - ASSERT(*ValueLocation(def_inst) == host_loc); + DEBUG_ASSERT(*ValueLocation(def_inst) == host_loc); } void RegAlloc::DefineValueImpl(BlockOfCode& code, IR::Inst* def_inst, const IR::Value& use_inst) noexcept { - ASSERT(!ValueLocation(def_inst) && "def_inst has already been defined"); + DEBUG_ASSERT(!ValueLocation(def_inst) && "def_inst has already been defined"); if (use_inst.IsImmediate()) { const HostLoc location = ScratchImpl(code, gpr_order); DefineValueImpl(code, def_inst, location); LoadImmediate(code, use_inst, location); } else { - ASSERT(ValueLocation(use_inst.GetInst()) && "use_inst must already be defined"); - const HostLoc location = *ValueLocation(use_inst.GetInst()); - DefineValueImpl(code, def_inst, location); + const auto location = ValueLocation(use_inst.GetInst()); + ASSERT(location && "use_inst must already be defined"); + DefineValueImpl(code, def_inst, *location); } } diff --git a/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.h b/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.h index 455c6a970a..0cad07796c 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.h +++ b/src/dynarmic/src/dynarmic/backend/x64/reg_alloc.h @@ -65,6 +65,11 @@ public: } void ReleaseOne() noexcept; void ReleaseAll() noexcept; + /// True when ReleaseAll would change nothing, as for most of the spill slots most of the time. + inline bool IsReleased() const noexcept { + return current_references == 0 && is_being_used_count == 0 && !is_scratch && !is_set_last_use && + (total_uses != accumulated_uses || (values.empty() && total_uses == 0 && max_bit_width == 0)); + } constexpr size_t GetMaxBitWidth() const noexcept { return 1 << max_bit_width; } void AddValue(HostLoc loc, IR::Inst* inst) noexcept; /// Checks if the given instruction is in our values set @@ -201,8 +206,10 @@ public: void ReleaseStackSpace(BlockOfCode& code, const size_t stack_space) noexcept; inline void EndOfAllocScope() noexcept { + // After every IR instruction: skip the locations it left alone. for (auto& iter : hostloc_info) - iter.ReleaseAll(); + if (!iter.IsReleased()) + iter.ReleaseAll(); } inline void AssertNoMoreUses() noexcept { ASSERT(std::all_of(hostloc_info.begin(), hostloc_info.end(), [](const auto& i) noexcept { return i.IsEmpty(); })); diff --git a/src/qt_common/game_list/worker.cpp b/src/qt_common/game_list/worker.cpp index 533ba1beb4..46460c1f66 100644 --- a/src/qt_common/game_list/worker.cpp +++ b/src/qt_common/game_list/worker.cpp @@ -419,7 +419,8 @@ void GameListWorker::ScanFileSystem(ScanTarget target, const std::vector<std::fi if (target == ScanTarget::PopulateGameList && (file_type == Loader::FileType::XCI || file_type == Loader::FileType::XCZ || - file_type == Loader::FileType::NSP || file_type == Loader::FileType::NSZ) && + file_type == Loader::FileType::NSP || file_type == Loader::FileType::NSZ || + file_type == Loader::FileType::NUP) && !Loader::IsBootableGameContainer(file, file_type)) { return; } diff --git a/src/qt_common/qt_common.cpp b/src/qt_common/qt_common.cpp index 975738dbe0..7ad7a477d2 100644 --- a/src/qt_common/qt_common.cpp +++ b/src/qt_common/qt_common.cpp @@ -58,7 +58,8 @@ std::unique_ptr<EmuThread> emu_thread = nullptr; const QStringList supported_file_extensions = {QStringLiteral("nro"), QStringLiteral("nso"), QStringLiteral("nca"), QStringLiteral("xci"), QStringLiteral("xcz"), QStringLiteral("nsp"), - QStringLiteral("nsz"), QStringLiteral("kip")}; + QStringLiteral("nsz"), QStringLiteral("nup"), + QStringLiteral("kip")}; Core::Frontend::WindowSystemType GetWindowSystemType() { // Determine WSI type based on Qt platform. diff --git a/src/shader_recompiler/CMakeLists.txt b/src/shader_recompiler/CMakeLists.txt index e60f26d47f..e4df9c045d 100644 --- a/src/shader_recompiler/CMakeLists.txt +++ b/src/shader_recompiler/CMakeLists.txt @@ -24,6 +24,7 @@ add_library(shader_recompiler STATIC backend/spirv/emit_spirv_memory.cpp backend/spirv/emit_spirv_select.cpp backend/spirv/emit_spirv_shared_memory.cpp + backend/spirv/emit_spirv_mesh.cpp backend/spirv/emit_spirv_special.cpp backend/spirv/emit_spirv_undefined.cpp backend/spirv/emit_spirv_warp.cpp diff --git a/src/shader_recompiler/backend/glasm/emit_glasm.cpp b/src/shader_recompiler/backend/glasm/emit_glasm.cpp index b795c01797..c18eec63cd 100644 --- a/src/shader_recompiler/backend/glasm/emit_glasm.cpp +++ b/src/shader_recompiler/backend/glasm/emit_glasm.cpp @@ -341,6 +341,9 @@ std::string_view StageHeader(Stage stage) { return "!!NVfp5.0\n"; case Stage::Compute: return "!!NVcp5.0\n"; + case Stage::Task: + case Stage::Mesh: + break; } throw InvalidArgument("Invalid stage {}", stage); } diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h b/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h index 6f522ab5c2..d66fe2f3d6 100644 --- a/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h +++ b/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h @@ -58,6 +58,10 @@ void EmitGetAttributeIndexed(EmitContext& ctx, IR::Inst& inst, ScalarS32 offset, void EmitSetAttributeIndexed(EmitContext& ctx, ScalarU32 offset, ScalarF32 value, ScalarU32 vertex); void EmitGetPatch(EmitContext& ctx, IR::Inst& inst, IR::Patch patch); void EmitSetPatch(EmitContext& ctx, IR::Patch patch, ScalarF32 value); +void EmitIsbeRead(EmitContext& ctx); +void EmitIsbeWriteAttribute(EmitContext& ctx); +void EmitIsbeWriteMapU8(EmitContext& ctx); +void EmitIsbeWriteMapU32(EmitContext& ctx); void EmitSetFragColor(EmitContext& ctx, u32 index, u32 component, ScalarF32 value); void EmitSetSampleMask(EmitContext& ctx, ScalarS32 value); void EmitSetFragDepth(EmitContext& ctx, ScalarF32 value); diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp b/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp index 1f4ffdd624..41c829ca5d 100644 --- a/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp +++ b/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp @@ -97,4 +97,20 @@ void EmitGetInBoundsFromOp(EmitContext& ctx) { NotImplemented(); } +void EmitIsbeRead(EmitContext& ctx) { + NotImplemented(); +} + +void EmitIsbeWriteAttribute(EmitContext& ctx) { + NotImplemented(); +} + +void EmitIsbeWriteMapU8(EmitContext& ctx) { + NotImplemented(); +} + +void EmitIsbeWriteMapU32(EmitContext& ctx) { + NotImplemented(); +} + } // namespace Shader::Backend::GLASM diff --git a/src/shader_recompiler/backend/glasm/glasm_emit_context.cpp b/src/shader_recompiler/backend/glasm/glasm_emit_context.cpp index 333a91cc58..02b6992b52 100644 --- a/src/shader_recompiler/backend/glasm/glasm_emit_context.cpp +++ b/src/shader_recompiler/backend/glasm/glasm_emit_context.cpp @@ -80,6 +80,9 @@ EmitContext::EmitContext(IR::Program& program, Bindings& bindings, const Profile case Stage::Compute: stage_name = "invocation"; break; + case Stage::Task: + case Stage::Mesh: + throw NotImplementedException("GLASM task and mesh programs"); } const std::string_view attr_stage{stage == Stage::Fragment ? "fragment" : "vertex"}; const VaryingState loads{info.loads.mask | info.passthrough.mask}; diff --git a/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h b/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h index 96aac502d5..06069d4c56 100644 --- a/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h +++ b/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h @@ -72,6 +72,10 @@ void EmitSetAttributeIndexed(EmitContext& ctx, std::string_view offset, std::str std::string_view vertex); void EmitGetPatch(EmitContext& ctx, IR::Inst& inst, IR::Patch patch); void EmitSetPatch(EmitContext& ctx, IR::Patch patch, std::string_view value); +void EmitIsbeRead(EmitContext& ctx); +void EmitIsbeWriteAttribute(EmitContext& ctx); +void EmitIsbeWriteMapU8(EmitContext& ctx); +void EmitIsbeWriteMapU32(EmitContext& ctx); void EmitSetFragColor(EmitContext& ctx, u32 index, u32 component, std::string_view value); void EmitSetSampleMask(EmitContext& ctx, std::string_view value); void EmitSetFragDepth(EmitContext& ctx, std::string_view value); diff --git a/src/shader_recompiler/backend/glsl/emit_glsl_not_implemented.cpp b/src/shader_recompiler/backend/glsl/emit_glsl_not_implemented.cpp index 9f1ed95a47..18779e51b8 100644 --- a/src/shader_recompiler/backend/glsl/emit_glsl_not_implemented.cpp +++ b/src/shader_recompiler/backend/glsl/emit_glsl_not_implemented.cpp @@ -97,4 +97,20 @@ void EmitGetInBoundsFromOp(EmitContext& ctx) { NotImplemented(); } +void EmitIsbeRead(EmitContext& ctx) { + NotImplemented(); +} + +void EmitIsbeWriteAttribute(EmitContext& ctx) { + NotImplemented(); +} + +void EmitIsbeWriteMapU8(EmitContext& ctx) { + NotImplemented(); +} + +void EmitIsbeWriteMapU32(EmitContext& ctx) { + NotImplemented(); +} + } // namespace Shader::Backend::GLSL diff --git a/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp b/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp index a2df6159fb..e4b11d6844 100644 --- a/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp +++ b/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp @@ -317,6 +317,9 @@ EmitContext::EmitContext(IR::Program& program, Bindings& bindings, const Profile header += "layout(early_fragment_tests)in;"; } break; + case Stage::Task: + case Stage::Mesh: + throw NotImplementedException("GLSL task and mesh programs"); case Stage::Compute: stage_name = "cs"; const u32 local_x{(std::max)(program.workgroup_size[0], 1u)}; diff --git a/src/shader_recompiler/backend/spirv/emit_spirv.cpp b/src/shader_recompiler/backend/spirv/emit_spirv.cpp index a903480c78..633ab25935 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv.cpp @@ -248,6 +248,20 @@ void DefineEntryPoint(const IR::Program& program, EmitContext& ctx, Id main) { case Stage::VertexB: execution_model = spv::ExecutionModel::Vertex; break; + case Stage::Task: + execution_model = spv::ExecutionModel::TaskEXT; + ctx.AddExecutionMode(main, spv::ExecutionMode::LocalSize, program.workgroup_size[0], 1U, + 1U); + break; + case Stage::Mesh: + execution_model = spv::ExecutionModel::MeshEXT; + ctx.AddExecutionMode(main, spv::ExecutionMode::LocalSize, program.workgroup_size[0], 1U, + 1U); + ctx.AddExecutionMode(main, spv::ExecutionMode::OutputVertices, program.mesh.max_vertices); + ctx.AddExecutionMode(main, spv::ExecutionMode::OutputPrimitivesEXT, + program.mesh.max_primitives); + ctx.AddExecutionMode(main, spv::ExecutionMode::OutputTrianglesEXT); + break; case Stage::TessellationControl: execution_model = spv::ExecutionModel::TessellationControl; ctx.AddCapability(spv::Capability::Tessellation); @@ -548,6 +562,51 @@ void RewriteOpcodes(std::vector<u32>& code, std::span<const std::pair<u32, spv:: offset += word_count; } } + +// Replaces the OpStore markers of the mesh instructions Sirit cannot emit: OpStore(vertices, +// primitives) becomes OpSetMeshOutputsEXT, and OpStore(x, y), OpStore(z, payload), OpReturn +// becomes the OpEmitMeshTasksEXT terminator. +void PatchMeshInstructions(std::vector<u32>& code, const EmitContext& ctx) { + const bool set_outputs{Sirit::ValidId(ctx.mesh_outputs_marker[0])}; + const bool emit_tasks{Sirit::ValidId(ctx.mesh_tasks_marker[0])}; + if (!set_outputs && !emit_tasks) { + return; + } + const auto store_at = [&](size_t offset, Id pointer, Id object) { + return offset + 2 < code.size() && + code[offset] == ((3U << 16) | static_cast<u32>(spv::Op::OpStore)) && + code[offset + 1] == pointer.value && code[offset + 2] == object.value; + }; + const auto& tasks{ctx.mesh_tasks_marker}; + size_t offset = 5; + bool patched{}; + while (offset < code.size()) { + const u32 word_count{code[offset] >> 16}; + if (word_count == 0) { + break; + } + if (set_outputs && + store_at(offset, ctx.mesh_outputs_marker[0], ctx.mesh_outputs_marker[1])) { + code[offset] = (3U << 16) | static_cast<u32>(spv::Op::OpSetMeshOutputsEXT); + patched = true; + } else if (emit_tasks && store_at(offset, tasks[0], tasks[1]) && + store_at(offset + 3, tasks[2], tasks[3]) && offset + 6 < code.size() && + code[offset + 6] == ((1U << 16) | static_cast<u32>(spv::Op::OpReturn))) { + const std::array<u32, 5> emit{(5U << 16) | static_cast<u32>(spv::Op::OpEmitMeshTasksEXT), + tasks[0].value, tasks[1].value, tasks[2].value, + tasks[3].value}; + code.erase(code.begin() + offset, code.begin() + offset + 7); + code.insert(code.begin() + offset, emit.begin(), emit.end()); + patched = true; + offset += emit.size(); + continue; + } + offset += word_count; + } + if (!patched) { + throw LogicError("Mesh instruction markers not found"); + } +} } // Anonymous namespace std::vector<u32> EmitSPIRV(const Profile& profile, const RuntimeInfo& runtime_info, IR::Program& program, Bindings& bindings) { @@ -564,6 +623,7 @@ std::vector<u32> EmitSPIRV(const Profile& profile, const RuntimeInfo& runtime_in PatchPhiNodes(program, ctx); std::vector<u32> code{ctx.Assemble()}; RewriteOpcodes(code, ctx.opcode_rewrites); + PatchMeshInstructions(code, ctx); return code; } diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h index 165ec3fe84..ec526ff354 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h +++ b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h @@ -61,6 +61,12 @@ Id EmitGetAttributeIndexed(EmitContext& ctx, Id offset, Id vertex); void EmitSetAttributeIndexed(EmitContext& ctx, Id offset, Id value, Id vertex); Id EmitGetPatch(EmitContext& ctx, IR::Patch patch); void EmitSetPatch(EmitContext& ctx, IR::Patch patch, Id value); +Id EmitIsbeRead(EmitContext& ctx, Id address); +void EmitIsbeWriteAttribute(EmitContext& ctx, Id address, Id value); +void EmitIsbeWriteMapU8(EmitContext& ctx, Id address, Id value); +void EmitIsbeWriteMapU32(EmitContext& ctx, const IR::Value& address, Id value); +void EmitMeshPrologue(EmitContext& ctx); +void EmitMeshEpilogue(EmitContext& ctx); void EmitSetFragColor(EmitContext& ctx, u32 index, u32 component, Id value); void EmitSetSampleMask(EmitContext& ctx, Id value); void EmitSetFragDepth(EmitContext& ctx, Id value); diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_mesh.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_mesh.cpp new file mode 100644 index 0000000000..6c599528cc --- /dev/null +++ b/src/shader_recompiler/backend/spirv/emit_spirv_mesh.cpp @@ -0,0 +1,227 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// Switch 2 (NVN) task and mesh programs. They talk to the hardware through the stage buffer +// (ISBE): a task program writes the number of mesh workgroups at byte 4 of its output region and +// its payload after it; each mesh workgroup reads that region (its own index at byte 0) and +// writes its vertices and primitives to an attribute region in groups of 32, one 32-word line per +// attribute, plus the primitive count and 8-bit vertex indices to a map region. Here the task's +// output region is the task payload, the mesh output regions live in workgroup memory, and the +// epilogue (reached by every invocation, see structured_control_flow.cpp) hands them to Vulkan. + +#include <algorithm> +#include <span> + +#include "shader_recompiler/backend/spirv/emit_spirv.h" +#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h" +#include "shader_recompiler/backend/spirv/spirv_emit_context.h" + +namespace Shader::Backend::SPIRV { +namespace { +void Barrier(EmitContext& ctx) { + ctx.OpControlBarrier(ctx.Const(static_cast<u32>(spv::Scope::Workgroup)), + ctx.Const(static_cast<u32>(spv::Scope::Workgroup)), + ctx.Const(static_cast<u32>(spv::MemorySemanticsMask::AcquireRelease | + spv::MemorySemanticsMask::WorkgroupMemory))); +} + +Id LocalInvocation(EmitContext& ctx) { + return ctx.OpCompositeExtract(ctx.U32[1], ctx.OpLoad(ctx.U32[3], ctx.local_invocation_id), 0U); +} + +Id Clamped(EmitContext& ctx, Id index, u32 count) { + return ctx.OpUMin(ctx.U32[1], index, ctx.Const(count - 1)); +} + +Id PayloadWord(EmitContext& ctx, Id index) { + return ctx.OpAccessChain(ctx.mesh_payload_u32, ctx.mesh_payload, + Clamped(ctx, index, ctx.mesh_info.payload_words)); +} + +Id SharedWord(EmitContext& ctx, Id array, Id index) { + return ctx.OpAccessChain(ctx.mesh_shared_u32, array, index); +} + +// Runs body under `if (condition)`. +template <typename Func> +void If(EmitContext& ctx, Id condition, Func&& body) { + const Id then_label{ctx.OpLabel()}; + const Id merge_label{ctx.OpLabel()}; + ctx.OpSelectionMerge(merge_label, spv::SelectionControlMask::MaskNone); + ctx.OpBranchConditional(condition, then_label, merge_label); + ctx.AddLabel(then_label); + body(); + ctx.OpBranch(merge_label); + ctx.AddLabel(merge_label); +} + +// Copies the lines of element `index` (a vertex or a primitive) from the attribute region. +void CopyLines(EmitContext& ctx, Id index, u32 base, u32 num_lines, + const std::vector<EmitContext::MeshLineTarget>& targets) { + // Word of line 0: base + (index / 32) * num_lines * 32 + index % 32. + const Id group{ctx.OpShiftRightLogical(ctx.U32[1], index, ctx.Const(5U))}; + const Id slot{ctx.OpBitwiseAnd(ctx.U32[1], index, ctx.Const(31U))}; + const Id first{ctx.OpIAdd(ctx.U32[1], + ctx.OpIMul(ctx.U32[1], group, ctx.Const(num_lines * 32)), + ctx.OpIAdd(ctx.U32[1], slot, ctx.Const(base)))}; + for (u32 line = 0; line < num_lines; ++line) { + const EmitContext::MeshLineTarget& target{targets[line]}; + if (!Sirit::ValidId(target.variable)) { + continue; + } + const Id word{ctx.OpLoad( + ctx.U32[1], + SharedWord(ctx, ctx.mesh_attributes, ctx.OpIAdd(ctx.U32[1], first, ctx.Const(line * 32))))}; + const Id value{target.is_float ? ctx.OpBitcast(ctx.F32[1], word) : word}; + const Id pointer_type{target.is_float ? ctx.output_f32 : ctx.output_u32}; + const Id pointer{target.element == ~0U + ? ctx.OpAccessChain(pointer_type, target.variable, index) + : ctx.OpAccessChain(pointer_type, target.variable, index, + ctx.Const(target.element))}; + ctx.OpStore(pointer, value); + } +} + +// OpenGL-style clip space: z = (z + w) / 2, from the stage buffer's position lines. +void ConvertDepth(EmitContext& ctx, Id vertex) { + const auto& lines{ctx.mesh_info.vertex_lines}; + const auto z_line{std::ranges::find(lines, static_cast<u32>(IR::Attribute::PositionZ))}; + const auto w_line{std::ranges::find(lines, static_cast<u32>(IR::Attribute::PositionW))}; + if (z_line == lines.end() || w_line == lines.end()) { + return; + } + const u32 num_lines{static_cast<u32>(lines.size())}; + const Id group{ctx.OpShiftRightLogical(ctx.U32[1], vertex, ctx.Const(5U))}; + const Id slot{ctx.OpBitwiseAnd(ctx.U32[1], vertex, ctx.Const(31U))}; + const Id first{ctx.OpIAdd(ctx.U32[1], ctx.OpIMul(ctx.U32[1], group, ctx.Const(num_lines * 32)), + slot)}; + const auto read = [&](auto line) { + const u32 offset{static_cast<u32>(line - lines.begin()) * 32}; + return ctx.OpBitcast(ctx.F32[1], + ctx.OpLoad(ctx.U32[1], SharedWord(ctx, ctx.mesh_attributes, + ctx.OpIAdd(ctx.U32[1], first, + ctx.Const(offset))))); + }; + const Id z{ctx.OpFMul(ctx.F32[1], ctx.OpFAdd(ctx.F32[1], read(z_line), read(w_line)), + ctx.Const(0.5f))}; + ctx.OpStore(ctx.OpAccessChain(ctx.output_f32, ctx.output_position, vertex, ctx.Const(2U)), z); +} + +void TaskEpilogue(EmitContext& ctx) { + Barrier(ctx); + const Id count{ctx.OpLoad(ctx.U32[1], PayloadWord(ctx, ctx.Const(1U)))}; + const Id one{ctx.Const(1U)}; + // Marker for OpEmitMeshTasksEXT count, 1, 1, payload (PatchMeshInstructions). + ctx.OpStore(count, one); + ctx.OpStore(one, ctx.mesh_payload); + ctx.mesh_tasks_marker = {count, one, one, ctx.mesh_payload}; +} + +void MeshEpilogue(EmitContext& ctx) { + const MeshInfo& mesh{ctx.mesh_info}; + Barrier(ctx); + const Id count{ctx.OpUMin(ctx.U32[1], ctx.OpLoad(ctx.U32[1], ctx.mesh_primitive_count), + ctx.Const(mesh.max_primitives))}; + const Id max_vertices{ctx.Const(mesh.max_vertices)}; + // Marker for OpSetMeshOutputsEXT max_vertices, count (PatchMeshInstructions). + ctx.OpStore(max_vertices, count); + ctx.mesh_outputs_marker = {max_vertices, count}; + + const Id invocation{LocalInvocation(ctx)}; + const u32 threads{ctx.mesh_threads}; + const bool convert_depth{ctx.runtime_info.convert_depth_mode && !ctx.profile.support_native_ndc && + Sirit::ValidId(ctx.output_position)}; + const u32 vertex_lines{static_cast<u32>(mesh.vertex_lines.size())}; + for (u32 first = 0; first < mesh.max_vertices; first += threads) { + const Id vertex{ctx.OpIAdd(ctx.U32[1], invocation, ctx.Const(first))}; + const auto copy = [&] { + CopyLines(ctx, vertex, 0, vertex_lines, ctx.mesh_vertex_targets); + if (convert_depth) { + ConvertDepth(ctx, vertex); + } + }; + if (first + threads > mesh.max_vertices) { + If(ctx, ctx.OpULessThan(ctx.U1, vertex, max_vertices), copy); + } else { + copy(); + } + } + const u32 primitive_lines{static_cast<u32>(mesh.primitive_lines.size())}; + for (u32 first = 0; first < mesh.max_primitives; first += threads) { + const Id primitive{ctx.OpIAdd(ctx.U32[1], invocation, ctx.Const(first))}; + If(ctx, ctx.OpULessThan(ctx.U1, primitive, count), [&] { + // Vertex indices: map bytes 4 + 3 * primitive. + const Id byte{ctx.OpIAdd(ctx.U32[1], ctx.OpIMul(ctx.U32[1], primitive, ctx.Const(3U)), + ctx.Const(4U))}; + std::array<Id, 3> indices; + for (u32 element = 0; element < 3; ++element) { + const Id address{ctx.OpIAdd(ctx.U32[1], byte, ctx.Const(element))}; + indices[element] = ctx.OpLoad(ctx.U32[1], SharedWord(ctx, ctx.mesh_map, address)); + } + const Id output_indices{ctx.TypePointer(spv::StorageClass::Output, ctx.U32[3])}; + ctx.OpStore(ctx.OpAccessChain(output_indices, ctx.mesh_primitive_indices, primitive), + ctx.OpCompositeConstruct(ctx.U32[3], std::span<const Id>(indices))); + CopyLines(ctx, primitive, ctx.mesh_primitive_base, primitive_lines, + ctx.mesh_primitive_targets); + }); + } +} +} // Anonymous namespace + +Id EmitIsbeRead(EmitContext& ctx, Id address) { + const Id workgroup{ + ctx.OpCompositeExtract(ctx.U32[1], ctx.OpLoad(ctx.U32[3], ctx.workgroup_id), 0U)}; + if (ctx.stage == Stage::Task) { + return workgroup; // the input region holds the workgroup index (VERTEX_ID) + } + const Id index{ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.Const(2U))}; + const Id word{ctx.OpLoad(ctx.U32[1], PayloadWord(ctx, index))}; + return ctx.OpSelect(ctx.U32[1], ctx.OpIEqual(ctx.U1, address, ctx.u32_zero_value), workgroup, + word); +} + +void EmitIsbeWriteAttribute(EmitContext& ctx, Id address, Id value) { + const Id index{ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.Const(2U))}; + if (ctx.stage == Stage::Task) { + ctx.OpStore(PayloadWord(ctx, index), value); + return; + } + ctx.OpStore(SharedWord(ctx, ctx.mesh_attributes, Clamped(ctx, index, ctx.mesh_attribute_words)), + value); +} + +void EmitIsbeWriteMapU8(EmitContext& ctx, Id address, Id value) { + if (ctx.stage != Stage::Mesh) { + throw NotImplementedException("Stage buffer map write outside mesh programs"); + } + ctx.OpStore(SharedWord(ctx, ctx.mesh_map, Clamped(ctx, address, ctx.mesh_map_bytes)), + ctx.OpBitwiseAnd(ctx.U32[1], value, ctx.Const(0xffU))); +} + +void EmitIsbeWriteMapU32(EmitContext& ctx, const IR::Value& address, Id value) { + // The primitive count: the map is written from its end, so its first word is at byte 3. + if (ctx.stage != Stage::Mesh || !address.IsImmediate() || address.U32() != 3) { + throw NotImplementedException("32-bit stage buffer map write"); + } + ctx.OpStore(ctx.mesh_primitive_count, value); +} + +void EmitMeshPrologue(EmitContext& ctx) { + if (ctx.stage != Stage::Mesh) { + return; + } + // Every invocation stores the zero: no branch here, the entry block's label is a phi + // predecessor for later blocks. + ctx.OpStore(ctx.mesh_primitive_count, ctx.u32_zero_value); + Barrier(ctx); +} + +void EmitMeshEpilogue(EmitContext& ctx) { + if (ctx.stage == Stage::Task) { + TaskEpilogue(ctx); + } else if (ctx.stage == Stage::Mesh) { + MeshEpilogue(ctx); + } +} + +} // namespace Shader::Backend::SPIRV diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_special.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_special.cpp index 8eca1fac29..2b36c1fd8d 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_special.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_special.cpp @@ -137,9 +137,16 @@ void EmitPrologue(EmitContext& ctx) { if (ctx.stage == Stage::VertexB || ctx.stage == Stage::Geometry) { SetFixedPipelinePointSize(ctx); } + if (ctx.stage == Stage::Task || ctx.stage == Stage::Mesh) { + EmitMeshPrologue(ctx); + } } void EmitEpilogue(EmitContext& ctx) { + if (ctx.stage == Stage::Task || ctx.stage == Stage::Mesh) { + EmitMeshEpilogue(ctx); + return; + } if (ctx.stage == Stage::VertexB && ctx.runtime_info.convert_depth_mode && !ctx.profile.support_native_ndc) { ConvertDepthMode(ctx); diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp index b562e82983..f80f0440df 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp @@ -257,6 +257,10 @@ std::string_view StageName(Stage stage) { return "fs"; case Stage::Compute: return "cs"; + case Stage::Task: + return "ts"; + case Stage::Mesh: + return "ms"; } throw InvalidArgument("Invalid stage {}", stage); } @@ -475,6 +479,7 @@ EmitContext::EmitContext(const Profile& profile_, const RuntimeInfo& runtime_inf DefineCommonTypes(program.info); DefineCommonConstants(); DefineInterfaces(program); + DefineMesh(program); DefineLocalMemory(program); DefineSharedMemory(program); DefineSharedMemoryFunctions(program); @@ -1586,6 +1591,16 @@ void EmitContext::DefineInputs(const IR::Program& program) { } const bool is_integer = input_type == AttributeType::SignedInt || input_type == AttributeType::UnsignedInt; + if (runtime_info.per_primitive_generics.test(index)) { + // Written per primitive by a mesh program: constant over the primitive. + AddExtension("SPV_EXT_mesh_shader"); + AddCapability(spv::Capability::MeshShadingEXT); + Decorate(id, spv::Decoration::PerPrimitiveEXT); + if (is_integer) { + Decorate(id, spv::Decoration::Flat); + } + continue; + } if (is_integer) { Decorate(id, spv::Decoration::Flat); } else { @@ -1614,8 +1629,120 @@ void EmitContext::DefineInputs(const IR::Program& program) { } } +void EmitContext::DefineMesh(const IR::Program& program) { + if (stage != Stage::Task && stage != Stage::Mesh) { + return; + } + AddExtension("SPV_EXT_mesh_shader"); + AddCapability(spv::Capability::MeshShadingEXT); + mesh_info = program.mesh; + mesh_threads = program.workgroup_size[0]; + // The task's output region, which its mesh workgroups read as their input region. + const Id payload_type{TypeArray(U32[1], Const(mesh_info.payload_words))}; + mesh_payload = DefineVariable(*this, payload_type, std::nullopt, + spv::StorageClass::TaskPayloadWorkgroupEXT); + Name(mesh_payload, "task_payload"); + mesh_payload_u32 = TypePointer(spv::StorageClass::TaskPayloadWorkgroupEXT, U32[1]); + if (stage == Stage::Task) { + return; + } + // Output attribute region: groups of 32 vertices, then groups of 32 primitives, one 32-word + // line per attribute. The map region keeps one map byte per word. + const u32 vertex_lines{static_cast<u32>(mesh_info.vertex_lines.size())}; + const u32 primitive_lines{static_cast<u32>(mesh_info.primitive_lines.size())}; + mesh_primitive_base = Common::DivCeil(mesh_info.max_vertices, 32U) * vertex_lines * 32; + mesh_attribute_words = (std::max)( + mesh_primitive_base + Common::DivCeil(mesh_info.max_primitives, 32U) * primitive_lines * 32, + 1U); + mesh_map_bytes = 4 + 3 * mesh_info.max_primitives; + mesh_shared_u32 = TypePointer(spv::StorageClass::Workgroup, U32[1]); + mesh_attributes = DefineVariable(*this, TypeArray(U32[1], Const(mesh_attribute_words)), + std::nullopt, spv::StorageClass::Workgroup); + Name(mesh_attributes, "isbe_attributes"); + mesh_map = DefineVariable(*this, TypeArray(U32[1], Const(mesh_map_bytes)), std::nullopt, + spv::StorageClass::Workgroup); + Name(mesh_map, "isbe_map"); + mesh_primitive_count = + DefineVariable(*this, U32[1], std::nullopt, spv::StorageClass::Workgroup); + Name(mesh_primitive_count, "isbe_primitive_count"); + + const Id vertex_count{Const(mesh_info.max_vertices)}; + const Id primitive_count{Const(mesh_info.max_primitives)}; + mesh_primitive_indices = + DefineVariable(*this, TypeArray(U32[3], primitive_count), + spv::BuiltIn::PrimitiveTriangleIndicesEXT, spv::StorageClass::Output); + std::array<Id, IR::NUM_GENERICS> generics[2]{}; + Id point_size{}, primitive_layer{}, primitive_viewport{}, primitive_id_output{}; + const auto per_primitive_builtin = [&](Id& variable, spv::BuiltIn builtin) { + if (!Sirit::ValidId(variable)) { + variable = DefineVariable(*this, TypeArray(U32[1], primitive_count), builtin, + spv::StorageClass::Output); + Decorate(variable, spv::Decoration::PerPrimitiveEXT); + } + return MeshLineTarget{variable, ~0U, false}; + }; + const auto target = [&](u32 line, bool per_primitive) -> MeshLineTarget { + const IR::Attribute attribute{line}; + if (IR::IsGeneric(attribute)) { + const u32 index{IR::GenericAttributeIndex(attribute)}; + Id& variable{generics[per_primitive][index]}; + if (!Sirit::ValidId(variable)) { + variable = DefineVariable( + *this, TypeArray(F32[4], per_primitive ? primitive_count : vertex_count), + std::nullopt, spv::StorageClass::Output); + Decorate(variable, spv::Decoration::Location, index); + if (per_primitive) { + Decorate(variable, spv::Decoration::PerPrimitiveEXT); + } + } + return {variable, IR::GenericAttributeElement(attribute), true}; + } + if (per_primitive) { + switch (attribute) { + case IR::Attribute::PrimitiveId: + return per_primitive_builtin(primitive_id_output, spv::BuiltIn::PrimitiveId); + case IR::Attribute::Layer: + return per_primitive_builtin(primitive_layer, spv::BuiltIn::Layer); + case IR::Attribute::ViewportIndex: + return per_primitive_builtin(primitive_viewport, spv::BuiltIn::ViewportIndex); + default: + return {}; + } + } + switch (attribute) { + case IR::Attribute::PositionX: + case IR::Attribute::PositionY: + case IR::Attribute::PositionZ: + case IR::Attribute::PositionW: + if (!Sirit::ValidId(output_position)) { + output_position = DefineVariable(*this, TypeArray(F32[4], vertex_count), + spv::BuiltIn::Position, spv::StorageClass::Output); + } + return {output_position, + static_cast<u32>(attribute) - static_cast<u32>(IR::Attribute::PositionX), true}; + case IR::Attribute::PointSize: + if (!Sirit::ValidId(point_size)) { + point_size = DefineVariable(*this, TypeArray(F32[1], vertex_count), + spv::BuiltIn::PointSize, spv::StorageClass::Output); + } + return {point_size, ~0U, true}; + default: + return {}; // per-primitive values in vertex lines have no vertex output + } + }; + for (const u32 line : mesh_info.vertex_lines) { + mesh_vertex_targets.push_back(target(line, false)); + } + for (const u32 line : mesh_info.primitive_lines) { + mesh_primitive_targets.push_back(target(line, true)); + } +} + void EmitContext::DefineOutputs(const IR::Program& program) { const Info& info{program.info}; + if (stage == Stage::Task || stage == Stage::Mesh) { + return; // DefineMesh + } const std::optional<u32> invocations{program.invocations}; if (runtime_info.convert_depth_mode || info.stores.AnyComponent(IR::Attribute::PositionX) || stage == Stage::VertexB) { diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.h b/src/shader_recompiler/backend/spirv/spirv_emit_context.h index a5821e9e93..1f9b534aaf 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.h +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.h @@ -363,6 +363,32 @@ public: std::vector<Id> interfaces; std::vector<std::pair<u32, spv::Op>> opcode_rewrites; + // Task and mesh programs. Their stage buffer: the task's output region is the task payload + // (the mesh stage's input region); the mesh output regions live in workgroup memory until + // the epilogue copies them to the outputs. The two mesh instructions Sirit lacks are emitted + // as OpStore markers with these operands and replaced after assembly (PatchMeshInstructions). + struct MeshLineTarget { + Id variable; ///< arrayed output + u32 element; ///< vector element, or ~0 for a scalar output + bool is_float; ///< bitcast the stage buffer word to F32 + }; + Id mesh_payload{}; + Id mesh_payload_u32{}; ///< pointer type to one payload word + Id mesh_attributes{}; + Id mesh_map{}; + Id mesh_primitive_count{}; + Id mesh_shared_u32{}; ///< pointer type to one workgroup word + Id mesh_primitive_indices{}; + MeshInfo mesh_info{}; + u32 mesh_threads{}; + u32 mesh_attribute_words{}; + u32 mesh_map_bytes{}; + u32 mesh_primitive_base{}; ///< first word of the per-primitive groups + std::vector<MeshLineTarget> mesh_vertex_targets; + std::vector<MeshLineTarget> mesh_primitive_targets; + std::array<Id, 2> mesh_outputs_marker{}; + std::array<Id, 4> mesh_tasks_marker{}; + Id load_const_func_u8{}; Id load_const_func_u16{}; Id load_const_func_u32{}; @@ -402,6 +428,7 @@ private: void DefineInputs(const IR::Program& program); void DefineOutputs(const IR::Program& program); + void DefineMesh(const IR::Program& program); }; } // namespace Shader::Backend::SPIRV diff --git a/src/shader_recompiler/environment.h b/src/shader_recompiler/environment.h index 67486430c8..20364944cf 100644 --- a/src/shader_recompiler/environment.h +++ b/src/shader_recompiler/environment.h @@ -66,6 +66,7 @@ struct ConstBufferAddr { IR::U32 dynamic_offset; u32 count; bool has_secondary; + u32 handle_program{~0U}; ///< index in Info::handle_programs for a handle computed per draw }; class Environment { @@ -121,9 +122,22 @@ public: return is_proprietary_driver; } + /// Task and mesh programs: SET_MESH_SHADER_A, SET_MESH_SHADER_B and SET_MESH_INIT_SHADER. + [[nodiscard]] const std::array<u32, 3>& MeshRegisters() const noexcept { + return mesh_registers; + } + + /// Mesh programs: the header of the pass-through geometry program bound with them, whose + /// output map lists the per-primitive outputs. + [[nodiscard]] const ProgramHeader& MeshPrimitiveHeader() const noexcept { + return mesh_primitive_header; + } + protected: ProgramHeader sph{}; std::array<u32, 8> gp_passthrough_mask{}; + std::array<u32, 3> mesh_registers{}; + ProgramHeader mesh_primitive_header{}; Stage stage{}; u32 start_address{}; bool is_proprietary_driver{}; diff --git a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h index 031f7fac75..024ed27ea3 100644 --- a/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h +++ b/src/shader_recompiler/frontend/ampere/lower_to_maxwell.h @@ -34,6 +34,9 @@ struct LoweredProgram { std::vector<u32> source_instruction_offsets; u32 source_size{}; u32 required_texture_buffer = ~0U; + // UR28 and up seen without a mirror: lowering again with this many registers below R216 + // reserved for them (even, so pairs stay adjacent) maps them. + u32 high_uniform_registers{}; u32 error_offset{}; std::string_view error; @@ -43,8 +46,10 @@ struct LoweredProgram { // Lower only operations whose semantics are representable by the existing // Maxwell frontend. These are new instructions, not reinterpretations of the // native bytes. Scheduling/control words are added by the environment adapter. -inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { +inline LoweredProgram LowerToMaxwellWith(std::span<const Instruction> code, u32 high_uniforms) { LoweredProgram result; + // UR28 and up (when used) live just below R216: UR28 at R(216 - high_uniforms). + const u32 high_uniform_base = 216 - high_uniforms; struct Relocation { u32 output_index; s64 target; }; std::vector<Relocation> relocations; std::vector<u32> barrier_tokens; @@ -55,8 +60,8 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { bool uses_uniform_bank = false; for (const auto& instruction : code) { const u32 op = instruction.Bits(0, 12); - uses_uniform_bank |= op == 0x3c4 || op == 0x2bd || op == 0xc0c || op == 0x986 || op == 0x9a8 || op == 0xc12 || op == 0xc17 || op == 0xe24 || op == 0xc24 || op == 0xd00 || op == 0xd09 || op == 0x9c3 || op == 0xc45 || op == 0x2a4 || op == 0x891 || op == 0x890 || op == 0x290 || op == 0x28c || op == 0x88c || op == 0x287 || op == 0x887 || op == 0x958 || op == 0xab9 || op == 0xabb || op == 0x899 || op == 0xc20 || op == 0xe21 || op == 0xe23 || op == 0xc02 || - op == 0x886 || op == 0x3c2 || op == 0xc82 || op == 0x882 || op == 0x292 || op == 0x892 || op == 0x291 || op == 0x981 || op == 0xc13 || op == 0xc10 || op == 0xc23 || op == 0x526 || op == 0x2be || op == 0x8a4 || op == 0xd05 || (op == 0x984 && instruction.Bits(91, 1)) || op == 0x89c || op == 0x883 || op == 0x2a5 || op == 0xd06 || op == 0x299 || op == 0xc09 || (op == 0x81c && instruction.Bits(67, 1)); + uses_uniform_bank |= op == 0x3c4 || op == 0x2bd || op == 0x896 || op == 0xf60 || op == 0xc11 || op == 0xd08 || op == 0xc0c || op == 0x986 || op == 0x9a8 || op == 0xc12 || op == 0xc17 || op == 0xe24 || op == 0xc24 || op == 0xd00 || op == 0xd09 || op == 0x9c3 || op == 0xc45 || op == 0x2a4 || op == 0x891 || op == 0x890 || op == 0x290 || op == 0x28c || op == 0x88c || op == 0x287 || op == 0x887 || op == 0x958 || op == 0xab9 || op == 0xabb || op == 0x899 || op == 0xc20 || op == 0xe21 || op == 0xe23 || op == 0xc02 || + op == 0x886 || op == 0x3c2 || op == 0xc82 || op == 0x882 || op == 0x292 || op == 0x892 || op == 0x291 || op == 0x981 || op == 0xc13 || op == 0xc10 || op == 0xc23 || op == 0x526 || op == 0x2be || op == 0x8a4 || op == 0xd05 || (op == 0x984 && instruction.Bits(91, 1)) || op == 0x89c || op == 0x883 || op == 0x2a5 || op == 0xd06 || op == 0xf63 || op == 0xf69 || op == 0xc0b || op == 0x8a5 || op == 0x299 || op == 0xc09 || (op == 0x81c && instruction.Bits(67, 1)) || op == 0xc0a; } const auto address_of = [](u32 index) { return 0x50U + index * 8 + ((index + 1) / 3) * 8; }; const auto fail = [&](u32 index, std::string_view reason) { @@ -92,8 +97,13 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { active_link = call_links[call]; } } - // RET has no destination GPR; bits 16..23 are reserved, not a write to R0. - if (active_link && opcode != 0x950 && insn.Bits(16, 8) == *active_link) + // RET, BRA, NOP and ISETP/FSETP (predicate outputs) have no destination GPR; bits 16..23 + // are not a write to R0. IMAD.WIDE writes a register pair. + const bool no_destination = opcode == 0x950 || opcode == 0x947 || opcode == 0x918 || + (opcode & 0x1ff) == 0x00c || (opcode & 0x1ff) == 0x00b; + const bool pair = opcode == 0x225 || opcode == 0x625 || opcode == 0x825; + if (active_link && !no_destination && + (insn.Bits(16, 8) == *active_link || (pair && insn.Bits(16, 8) + 1 == *active_link))) return fail(index, "native callee overwrites link register"); } // Scratch values and emulated uniform state occupy reserved high GPRs. @@ -101,22 +111,24 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { for (u32 bit : {16U, 24U, 32U, 64U}) { if (bit == 32 && insn.Bits(9, 3) != 1 && opcode != 0xb60 && opcode != 0xb66) continue; const u32 reg = insn.Bits(bit, 8); - if (reg >= (uses_uniform_bank ? 216U : 246U) && reg < 255) return fail(index, "native program uses lowering scratch registers"); + if (reg >= (uses_uniform_bank ? high_uniform_base : 246U) && reg < 255) return fail(index, "native program uses lowering scratch registers"); } // UR0-UR21 live in R216-R237 and UR22-UR27 in R240-R245 (R239 holds the uniform - // predicates), so even/odd pairs stay adjacent. Sites that still check reg < 16 rely on - // UR0-UR15 being R216-R231. + // predicates), so even/odd pairs stay adjacent; UR28 and up below R216. Sites that + // still check reg < 16 rely on UR0-UR15 being R216-R231. const auto uniform_register = [&](u32 reg) -> u32 { if (reg < 22) return 216 + reg; if (reg < 28) return 218 + reg; if (reg == 63) return 255; + if (reg < 63 && reg - 28 < high_uniforms) return high_uniform_base + (reg - 28); + if (reg < 63) result.high_uniform_registers = (std::max)(result.high_uniform_registers, (reg - 26) & ~1U); return ~0U; }; bool rewritten_uniform = false; bool uniform_guard = false; u32 uniform_output = 7; // ULOP3's UPn output: computed in P(6-n), then merged into R239 if (opcode == 0x2a4 || opcode == 0x8a4 || opcode == 0x891 || opcode == 0x890 || opcode == 0x290 || - opcode == 0x292 || opcode == 0x892 || opcode == 0x291) { + opcode == 0x292 || opcode == 0x892 || opcode == 0x291 || opcode == 0x896 || opcode == 0x8a5) { if (insn.Bits(12, 4) != 7 && opcode != 0x292 && opcode != 0x892) return fail(index, "native uniform arithmetic predication"); uniform_guard = insn.Bits(12, 4) != 7; @@ -138,7 +150,8 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { insn.words[2] = (insn.words[2] & ~(7U << 17)) | ((6 - uniform_output) << 17); } opcode = opcode == 0x2a4 ? 0x224 : opcode == 0x8a4 ? 0x824 : opcode == 0x891 ? 0x811 : opcode == 0x890 ? 0x810 - : opcode == 0x292 ? 0x212 : opcode == 0x892 ? 0x812 : opcode == 0x291 ? 0x211 : 0x210; + : opcode == 0x292 ? 0x212 : opcode == 0x892 ? 0x812 : opcode == 0x291 ? 0x211 + : opcode == 0x896 ? 0x816 : opcode == 0x8a5 ? 0x825 : 0x210; } const bool uniform_predication = opcode == 0xabb || opcode == 0xab9 || opcode == 0xc82 || opcode == 0x882; const u32 native_predicate = insn.Bits(12, 4); @@ -163,6 +176,13 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { }; std::string_view operand_error; const auto emit = [&](u64 word) { result.instructions.push_back(word); }; + // Bound Maxwell texture instructions share one descriptor constant buffer per program. + const auto bind_texture_buffer = [&](u32 binding) { + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return false; + result.required_texture_buffer = binding; + return true; + }; // P(guard slot) while a uniform guard is swapped in; P6 for lowerings that use it as // scratch (R2UR, MATCH, HFMA2.RELU), which save and restore it with the defaults below. const u32 guard_mask = guard_slot < 7 ? 1U << guard_slot : 0U; @@ -466,6 +486,20 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{insn.Bits(64, 8)} << 20)); break; } + if (insn.Bits(24, 8) == 255 && insn.words[1] == 255 && (insn.words[2] & ~0x07800affU) == 0x000e0400 && + (!rewritten_uniform || insn.Bits(87, 3) == 7)) { + // IMAD.X RZ, RZ, c, Pn: the high word of a 64-bit add (bit 75 inverts c). + u32 c = insn.Bits(64, 8); + mov_imm(250, 1); + emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + if (insn.Bits(75, 1)) { + emit(0x0400000000000000ULL | predicate | 251 | (u64{c} << 8) | (u64{0xffffffffU} << 20) | (u64{2} << 53)); + c = 251; + } + emit(base(0x5c10000000000000ULL) | (u64{c} << 8) | (u64{250} << 20)); + break; + } if ((insn.words[2] & ~0x2ffU) != 0x078e0000 || (insn.words[1] & ~0xffU)) return fail(index, "native integer register multiply mode"); multiply_low(insn.Bits(24, 8), insn.Bits(32, 8), insn.Bits(64, 8), false); @@ -496,16 +530,87 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(0x5c10000000070000ULL | d_low | (u64{250} << 8) | (u64{255} << 20)); break; } - case 0x827: { // IMAD.HI.U32, captured zero addend - if (insn.words[2] != 0x078e00ff) return fail(index, "native high multiply mode"); - const u32 a = insn.Bits(24, 8), b = insn.words[1]; + case 0x226: { // IDP.2A.LO.U16.U8: a.lo16 * b.byte0 + a.hi16 * b.byte1 + c + if ((insn.words[2] & ~0xffU) != 0x1000 || (insn.words[1] & ~0xffU)) + return fail(index, "native packed dot-product mode"); + const u32 a = insn.Bits(24, 8), b = insn.Bits(32, 8), c = insn.Bits(64, 8); + for (u32 component = 0; component < 2; ++component) { + emit(0x3828000000000000ULL | predicate | (250 + component) | (u64{b} << 8) | (u64{component * 8} << 20)); + emit(0x0400000000000000ULL | predicate | (250 + component) | (u64{250 + component} << 8) | (u64{0xff} << 20)); + } + xmad(252, a, 250, c, false, false, 0, false); + xmad(static_cast<u32>(dest), a, 251, 252, true, false, 0, false); + break; + } + case 0x225: case 0x227: case 0x625: case 0x825: case 0x827: case 0xa27: { + // IMAD.HI Rd / IMAD.WIDE Rd:Rd+1 = a * b + c:c+1 (.U32 or signed, bit 73), with an optional + // carry-out (bits 81-83). b is a register, an immediate (0x8xx) or a cbuf (0xa27); 0x625 takes + // b from bits 64-71 and adds a cbuf pair. + const bool wide = opcode == 0x225 || opcode == 0x625 || opcode == 0x825; + const bool immediate = opcode == 0x825 || opcode == 0x827, signed_product = insn.Bits(73, 1); + const u32 c = opcode == 0x625 ? 255U : insn.Bits(64, 8), carry_output = insn.Bits(81, 3); + if ((insn.words[2] & ~0x000e02ffU) != 0x07800000 || + ((opcode == 0x225 || opcode == 0x227) && (insn.words[1] & ~0xffU)) || + ((opcode == 0xa27 || opcode == 0x625) && (!valid_cbuf() || insn.Bits(62, 2))) || + (opcode == 0x625 && insn.Bits(38, 16) > 0xfff8) || (c != 255 && (c & 1)) || + (wide && (dest == 255 || (dest & 1))) || (signed_product && carry_output != 7)) + return fail(index, "native high multiply mode"); + const u32 a = insn.Bits(24, 8); + const u32 b = immediate ? insn.words[1] : opcode == 0xa27 ? 254U : insn.Bits(opcode == 0x625 ? 64 : 32, 8); + if (opcode == 0xa27) emit(0x4c98000000000000ULL | predicate | 254 | (u64{15} << 39) | cbuf()); + if (signed_product) { + // The signed high word is the unsigned one minus (a < 0 ? b : 0) + (b < 0 ? a : 0), + // which R253 holds (R252 is free until the multiplication). + const u32 b_register = immediate ? 254U : b; + if (immediate) mov_imm(254, b); + emit(0x3828000000000000ULL | predicate | 253 | (u64{a} << 8) | (u64{31} << 20) | (u64{1} << 48)); + emit(0x5c40000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{b_register} << 20) | (u64{7} << 48)); + emit(0x3828000000000000ULL | predicate | 252 | (u64{b_register} << 8) | (u64{31} << 20) | (u64{1} << 48)); + emit(0x5c40000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{a} << 20) | (u64{7} << 48)); + emit(0x5c10000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{252} << 20)); + } // Base-2^16 long multiplication. Each intermediate sum fits U32. - xmad(250, a, b, 255, false, false, 0, true); - xmad(251, a, b, 250, true, false, 2, true); - xmad(250, a, b, 251, false, true, 1, true); - xmad(252, a, b, 251, true, true, 2, true); + xmad(250, a, b, 255, false, false, 0, immediate); + xmad(251, a, b, 250, true, false, 2, immediate); + xmad(250, a, b, 251, false, true, 1, immediate); + xmad(252, a, b, 251, true, true, 2, immediate); emit(0x3828000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{16} << 20)); - emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20)); + if (signed_product) + emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{253} << 20) | (u64{1} << 48)); + if (!wide && c == 255 && carry_output == 7) { + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20)); + break; + } + // High word in R253 and low word in R250 (as UIMAD.WIDE.U32), then c:c+1 is added. + emit(0x5c10000000000000ULL | predicate | 253 | (u64{250} << 8) | (u64{252} << 20)); + xmad(250, a, b, 255, false, false, 0, immediate); + xmad(251, a, b, 255, true, false, 0, immediate); + xmad(251, a, b, 251, false, true, 0, immediate); + emit(0x5c18000000000000ULL | predicate | 250 | (u64{251} << 8) | (u64{250} << 20) | (u64{16} << 39)); + u32 c_low = c, c_high = c == 255 ? 255U : c + 1; + if (opcode == 0x625) { + emit(0x4c98000000000000ULL | predicate | 252 | (u64{15} << 39) | cbuf()); + emit(0x4c98000000000000ULL | predicate | 254 | (u64{15} << 39) | (cbuf() + (u64{1} << 20))); + c_low = 252; + c_high = 254; + } + const u32 high = wide ? static_cast<u32>(dest) + 1 : static_cast<u32>(dest); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{c_low} << 20) | (u64{1} << 47)); + if (carry_output == 7) { + emit(0x5c10000000000000ULL | predicate | high | (u64{253} << 8) | (u64{c_high} << 20) | (u64{1} << 43)); + if (wide) emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{255} << 20)); + break; + } + // Maxwell IADD has no .X with .CC: R251 takes the low carry, R254 counts the high word's. + emit(0x5c10000000000000ULL | predicate | 251 | (u64{255} << 8) | (u64{255} << 20) | (u64{1} << 43)); + emit(0x5c10000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{c_high} << 20) | (u64{1} << 47)); + emit(0x5c10000000000000ULL | predicate | 254 | (u64{255} << 8) | (u64{255} << 20) | (u64{1} << 43)); + emit(0x5c10000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{251} << 20) | (u64{1} << 47)); + emit(0x5c10000000000000ULL | predicate | 254 | (u64{254} << 8) | (u64{255} << 20) | (u64{1} << 43)); + emit(0x5c98000000000000ULL | predicate | high | (u64{253} << 20) | (u64{15} << 39)); + if (wide) emit(base(0x5c98000000000000ULL) | (u64{250} << 20) | (u64{15} << 39)); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{carry_output} << 3) | (u64{254} << 8) | + (u64{255} << 20) | (u64{7} << 39) | (u64{5} << 49)); break; } case 0xa24: { // IMAD, constant-buffer multiplier @@ -547,6 +652,21 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } case 0x824: { // Captured low-word IMAD forms; bit 75 negates the addend. const bool negate_c = insn.Bits(75, 1) != 0; + if (insn.words[1] == 1 && (insn.words[2] & ~(0x07800800U | 0xffU)) == 0x000e0600U) { + // IMAD.X Rd, Ra, 1, Rc, Pn: Ra + Rc plus the carry predicate in bits 87-90 (~Rc with bit 75). + u32 c = insn.Bits(64, 8); + mov_imm(252, 1); + emit(0x5ca0000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{255} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + if (negate_c) { + emit(0x0400000000000000ULL | predicate | 251 | (u64{c} << 8) | (u64{0xffffffffU} << 20) | (u64{2} << 53)); + c = 251; + } + emit(0x5c10000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | + (u64{c} << 20)); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20)); + break; + } if (insn.words[1] == 1 && (insn.words[2] & 0xfffff700U) == 0x078e0200) { emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(64, 8)} << 20) | (u64{negate_c} << 48)); @@ -614,8 +734,15 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(0x5c30000000070000ULL | 250 | (u64{250} << 20)); emit(0xef10000000070000ULL | 251 | (u64{insn.Bits(24, 8)} << 8) | (u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); - emit(0x5ca0000000070000ULL | target | (u64{251} << 8) | - (u64{target} << 20) | (u64{6} << 39)); + if (native_predicate == 7) { + // Every lane executing it is in the ballot, so no lane-less fallback can be taken. + // Leaving the old value selectable would make it a source of the result for the + // texture and global memory tracking (a handle traced to the uniform's last load). + emit(0x5c98000000070000ULL | target | (u64{15} << 39) | (u64{251} << 20)); + } else { + emit(0x5ca0000000070000ULL | target | (u64{251} << 8) | + (u64{target} << 20) | (u64{6} << 39)); + } restore_predicate(); break; } @@ -708,36 +835,42 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } case 0x899: { // USHF.L.U32; accept a zero word outside the selected half. const u32 dst = insn.Bits(16, 8), low = insn.Bits(24, 8), high = insn.Bits(64, 8); - if (insn.words[2] == (0x08011400U | high) && low == 63 && high < 16 && - dst < 16 && native_predicate == 7 && insn.words[1] <= 32) { - emit(0x3828000000000000ULL | predicate | (dst + 216) | - (u64{high + 216} << 8) | (u64{insn.words[1]} << 20) | (u64{1} << 48)); + const u32 dst_reg = uniform_register(dst), low_reg = uniform_register(low); + const u32 high_reg = uniform_register(high); + const bool dst_ok = dst != 63 && dst_reg != ~0U; + const bool low_ok = low != 63 && low_reg != ~0U, high_ok = high != 63 && high_reg != ~0U; + if (insn.words[2] == (0x08011400U | high) && low == 63 && high_ok && + dst_ok && native_predicate == 7 && insn.words[1] <= 32) { + emit(0x3828000000000000ULL | predicate | dst_reg | + (u64{high_reg} << 8) | (u64{insn.words[1]} << 20) | (u64{1} << 48)); break; } // Logical right shifts of one register (bit 76 right, bit 80 high word, bits 73-74 // U32): USHF.R.U32 URd, URa, n, URZ and USHF.R.U32.HI URd, URZ, n, URb. - const bool right_low = insn.words[2] == 0x0800163f && low < 16; - const bool right_high = insn.words[2] == (0x08011600U | high) && low == 63 && high < 16; - if ((right_low || right_high) && dst < 16 && native_predicate == 7 && insn.words[1] <= 32) { - emit(0x3828000000000000ULL | predicate | (dst + 216) | - (u64{(right_high ? high : low) + 216} << 8) | (u64{insn.words[1]} << 20)); + const bool right_low = insn.words[2] == 0x0800163f && low_ok; + const bool right_high = insn.words[2] == (0x08011600U | high) && low == 63 && high_ok; + if ((right_low || right_high) && dst_ok && native_predicate == 7 && insn.words[1] <= 32) { + emit(0x3828000000000000ULL | predicate | dst_reg | + (u64{right_high ? high_reg : low_reg} << 8) | (u64{insn.words[1]} << 20)); break; } - const bool high_result = insn.words[2] == (0x08010600U | high) && low == 63 && high < 16; - const bool low_result = insn.words[2] == 0x0800063f && low < 16; - if (dst >= 16 || native_predicate != 7 || insn.words[1] > 32 || (!high_result && !low_result)) + const bool high_result = insn.words[2] == (0x08010600U | high) && low == 63 && high_ok; + const bool low_result = insn.words[2] == 0x0800063f && low_ok; + if (!dst_ok || native_predicate != 7 || insn.words[1] > 32 || (!high_result && !low_result)) return fail(index, "native uniform shift mode"); - emit(0x3848000000000000ULL | predicate | (dst + 216) | - (u64{(high_result ? high : low) + 216} << 8) | (u64{insn.words[1]} << 20)); + emit(0x3848000000000000ULL | predicate | dst_reg | + (u64{high_result ? high_reg : low_reg} << 8) | (u64{insn.words[1]} << 20)); break; } - case 0xb82: { // LDC.64 indexed byte offset - const u32 width = insn.Bits(73, 3); - if ((width != 4 && width != 5) || insn.Bits(78, 2) || insn.Bits(91, 1) || + case 0xb82: { // LDC.64 indexed byte offset; bits 78-79 the mode as Maxwell's (2 = IS) + // .IS: Ra + offset addresses all constant buffers as one space, the buffer in its high + // half (Kirby Air Riders' vertex shaders), which Eden's Maxwell LDC implements. + const u32 width = insn.Bits(73, 3), mode = insn.Bits(78, 2); + if ((width != 4 && width != 5) || (mode != 0 && mode != 2) || insn.Bits(91, 1) || bank() >= 18 || (width == 5 && ((dest & 1) || dest >= 254))) return fail(index, "native indexed constant load mode"); emit(base(0xef90000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | - (u64{insn.Bits(38, 16)} << 20) | (u64{bank()} << 36) | (u64{width} << 48)); + (u64{insn.Bits(38, 16)} << 20) | (u64{bank()} << 36) | (u64{mode} << 44) | (u64{width} << 48)); break; } case 0xf89: { // SHFL with immediate lane and segmentation/clamp mask, any mode. @@ -759,33 +892,36 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { (u64{insn.Bits(64, 8)} << 20) | (u64{maxwell} << 28)); break; } - case 0xb6c: { // TXD 2D or ARRAY_2D, four channels; the two destination pairs anywhere. + case 0xb6c: { // TXD 2D or ARRAY_2D, any channels; the two destination pairs anywhere. const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); const u32 coordinates = insn.Bits(24, 8), derivatives = insn.Bits(32, 8); // Ampere dimensions: 1 = 2D, 5 = ARRAY_2D; Maxwell TXD's are 2 and 3. The coordinates // (x, y[, layer]) and the four gradients sit in the same registers for both. const u32 dimension = insn.Bits(61, 3); const u32 high = insn.Bits(64, 8); // destination of the third and fourth channels - if ((insn.words[2] & ~0xffU) != 0x001e0f00 || insn.Bits(59, 2) || + const u32 mask = insn.Bits(72, 4), channels = static_cast<u32>(std::popcount(mask)); + // Bit 90 is forwarded below (Maxwell's NODEP); Splatoon Raiders sets it. + if ((insn.words[2] & ~(0xfffU | (1U << 26))) != 0x001e0000 || mask == 0 || insn.Bits(59, 2) || (dimension != 1 && dimension != 5) || binding >= 18 || offset > 0x1fff || - (dest & 1) || dest >= 248 || (high & 1) || (high >= 248 && high != 255) || + (channels > 1 && (dest & 1)) || dest >= 248 || + (channels > 2 ? high != 255 && ((high & 1) || high >= 248) : high != 255) || coordinates >= 248 || derivatives >= 247) return fail(index, "native gradient texture mode"); if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) return fail(index, "native texture descriptor buffer conflict"); result.required_texture_buffer = binding; - // Maxwell writes four consecutive registers: when the second pair is elsewhere (Splatoon + // Maxwell writes consecutive registers: when the second pair is elsewhere (Splatoon // Raiders: R12/R18, R18/R44), write them to scratch and move each pair into place. - const bool consecutive = high == dest + 2; + const bool consecutive = channels <= 2 || high == dest + 2; const u64 target = consecutive ? dest : 248; emit(0xde00000000000000ULL | predicate | target | (u64{coordinates} << 8) | - (u64{derivatives} << 20) | (u64{dimension == 5 ? 3U : 2U} << 28) | (u64{15} << 31) | + (u64{derivatives} << 20) | (u64{dimension == 5 ? 3U : 2U} << 28) | (u64{mask} << 31) | (u64{offset} << 36) | (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51)); if (!consecutive) { - for (u32 i = 0; i < 2; ++i) { - emit(0x5c98000000000000ULL | predicate | (dest + i) | (u64{15} << 39) | (u64{248 + i} << 20)); - if (high != 255) - emit(0x5c98000000000000ULL | predicate | (high + i) | (u64{15} << 39) | (u64{250 + i} << 20)); + for (u32 i = 0; i < channels; ++i) { + if (i >= 2 && high == 255) break; + emit(0x5c98000000000000ULL | predicate | (i < 2 ? dest + i : high + i - 2) | + (u64{15} << 39) | (u64{248 + i} << 20)); } } break; @@ -836,44 +972,74 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(0x3828000000000000ULL | predicate | (dest + 216) | (u64{insn.words[1] + 216} << 8) | (u64{1} << 39) | (u64{1} << 40)); break; - case 0x3a1: { // MATCH.ALL.U64 with only the all-equal predicate retained. + case 0x2bf: // UPOPC, as POPC of the uniform's register. + if (dest >= 16 || insn.words[1] >= 16 || insn.words[2] != 0x08000000 || native_predicate != 7) + return fail(index, "native uniform population count mode"); + emit(0x5c08000000000000ULL | predicate | (dest + 216) | (u64{insn.words[1] + 216} << 20)); + break; + case 0x3a1: { // MATCH.ALL (.U32, or .U64 on bit 73) with only the all-equal predicate retained. const u32 source = insn.Bits(24, 8), target = insn.Bits(81, 3); - if (dest != 255 || source >= 254 || (source & 1) || target >= 6 || - native_predicate != 7 || insn.words[1] || (insn.words[2] & ~0xe0000U) != 0x200) + const u32 words = insn.Bits(73, 1) ? 2 : 1; + const u32 scratch = target == 6 ? 5U : 6U; // the comparison, saved around the match + if (dest != 255 || source >= 254 || (words == 2 && (source & 1)) || target == 7 || + native_predicate != 7 || insn.words[1] || (insn.words[2] & ~0xe0200U) != 0) return fail(index, "native warp match mode"); - save_predicate(); + save_predicate(1U << scratch); + // Compare every lane with the lowest active one, then require all to agree. emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48)); emit(0x5c10000000070000ULL | 251 | (u64{250} << 8) | (u64{255} << 20) | (u64{1} << 49)); emit(0x5c40000000070000ULL | 250 | (u64{250} << 8) | (u64{251} << 20) | (u64{7} << 48)); emit(0x5c30000000070000ULL | 250 | (u64{250} << 20)); - for (u32 word = 0; word < 2; ++word) + for (u32 word = 0; word < words; ++word) emit(0xef10000000070000ULL | (251 + word) | (u64{source + word} << 8) | (u64{250} << 20) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); - emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source} << 8) | + emit(0x5b60000000070000ULL | 7 | (u64{scratch} << 3) | (u64{source} << 8) | (u64{251} << 20) | (u64{7} << 39) | (u64{2} << 49)); - emit(0x5b60000000070000ULL | 7 | (u64{6} << 3) | (u64{source + 1} << 8) | - (u64{252} << 20) | (u64{6} << 39) | (u64{2} << 49)); - emit(0x50d8000000070000ULL | 255 | (u64{6} << 39) | (u64{target} << 45)); - restore_predicate(); - break; - } - case 0x3c4: { // REDUX.SUM (modulo 2^32) or .MAX (unsigned) over the actual active lanes. + if (words == 2) + emit(0x5b60000000070000ULL | 7 | (u64{scratch} << 3) | (u64{source + 1} << 8) | + (u64{252} << 20) | (u64{scratch} << 39) | (u64{2} << 49)); + emit(0x50d8000000070000ULL | 255 | (u64{scratch} << 39) | (u64{target} << 45)); + restore_predicate(1U << scratch); + break; + } + case 0x80d: { // CSMTEST: a test of shading-mode state Eden does not model, taken as false. + // Kirby Air Riders' vertex-side shaders run "CSMTEST.A.FCSM_TR P0" then "@P0 EXIT": + // never leave early. Outputs of PT discard the result. + const u32 first = insn.Bits(81, 3), second = insn.Bits(84, 3); + if (insn.Bits(87, 4) != 7 || insn.Bits(16, 16)) + return fail(index, "native CSM test mode"); + for (const u32 target : {first, second}) + if (target != 7) + emit(0x5090000000000000ULL | predicate | (u64{target} << 3) | 7 | (u64{7} << 12) | + (u64{1} << 15) | (u64{7} << 29) | (u64{7} << 39)); + break; + } + case 0x3c4: { // REDUX.SUM (modulo 2^32), .MIN/.MAX (unsigned), .OR or .XOR over the actual active lanes. const u32 source = insn.Bits(24, 8), target = uniform_register(static_cast<u32>(dest)); - const bool maximum = insn.words[2] == 0x14000; + const bool maximum = insn.words[2] == 0x14000, minimum = insn.words[2] == 0x10000; + // OR and XOR: inactive lanes contribute zero, as for the sum. Maxwell LOP: 1 = OR, 2 = XOR. + const u32 logical = insn.words[2] == 0x4000 ? 1U : insn.words[2] == 0x8000 ? 2U : 0U; if (target == ~0U || target == 255 || insn.words[1] || - (insn.words[2] != 0xc000 && !maximum) || native_predicate != 7) + (insn.words[2] != 0xc000 && !maximum && !minimum && !logical) || native_predicate != 7) return fail(index, "native warp sum mode"); emit(0x50d8000000070000ULL | 250 | (u64{7} << 39) | (u64{7} << 45) | (u64{1} << 48)); - mov_imm(target, 0); + mov_imm(target, minimum ? ~0U : 0U); for (u32 lane = 0; lane < 32; ++lane) { emit(0xef10000000070000ULL | 251 | (u64{source} << 8) | (u64{lane} << 20) | (u64{1} << 28) | (u64{1} << 29) | (u64{31} << 34) | (u64{7} << 48)); emit(0x3828000000070000ULL | 252 | (u64{250} << 8) | (u64{lane} << 20)); emit(0x0400000000070000ULL | 252 | (u64{252} << 8) | (u64{1} << 20)); emit(0x5c10000000070000ULL | 252 | (u64{255} << 8) | (u64{252} << 20) | (u64{1} << 48)); - emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | (u64{7} << 48)); - if (maximum) { + // Inactive lanes: 0, or ~0 for the minimum (OR with the inverted lane mask). + emit(0x5c40000000070000ULL | 251 | (u64{251} << 8) | (u64{252} << 20) | (u64{7} << 48) | + (u64{minimum} << 40) | (u64{minimum} << 41)); + if (minimum) { + emit(0x5c20000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) | (u64{7} << 39)); + } else if (maximum) { emit(0x5c20000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) | (u64{15} << 39)); + } else if (logical) { + emit(0x5c40000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20) | + (u64{logical} << 41) | (u64{7} << 48)); } else { emit(0x5c10000000070000ULL | target | (u64{target} << 8) | (u64{251} << 20)); } @@ -881,8 +1047,9 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { break; } case 0xf8c: { // ATOMS.POPC.INC.32 with no returned value: one increment per active lane. - const u32 stride = insn.Bits(78, 2); - if (dest != 255 || insn.words[1] || (insn.words[2] & ~0xc000U) != 0x0d00003f) + const u32 stride = insn.Bits(78, 2), offset = insn.Bits(40, 24); + if (dest != 255 || (insn.words[1] & 0xff) || (offset & 3) || + (insn.words[2] & ~0xc000U) != 0x0d00003f) return fail(index, "native shared increment mode"); u32 address = insn.Bits(24, 8); if (stride && address != 255) { @@ -891,15 +1058,18 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { address = 251; } mov_imm(250, 1); - emit(0xec00000000000000ULL | predicate | 255 | (u64{address} << 8) | (u64{250} << 20)); + emit(0xec00000000000000ULL | predicate | 255 | (u64{address} << 8) | (u64{250} << 20) | + (u64{offset >> 2} << 30)); break; } - case 0x38c: { // ATOMS.ADD.U32, naturally aligned shared byte offset. - const u32 offset = insn.Bits(40, 24); - if (insn.words[2] || (offset & 3)) return fail(index, "native shared atomic mode"); + case 0x38c: { // ATOMS (ADD/MIN/MAX/INC/DEC/AND/OR/XOR/EXCH, U32 or S32), aligned byte offset. + // The operation (bits 87-90) is numbered as in Maxwell ATOMS (52-55); bit 73 is S32. + const u32 offset = insn.Bits(40, 24), operation = insn.Bits(87, 4); + if ((insn.words[2] & ~0x07800200U) || operation > 8 || (offset & 3)) + return fail(index, "native shared atomic mode"); emit(0xec00000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) | - (u64{offset >> 2} << 30)); + (u64{insn.Bits(73, 1)} << 28) | (u64{offset >> 2} << 30) | (u64{operation} << 52)); break; } case 0x3a8: { // ATOMG.E.ADD.STRONG.GPU U32 at a 64-bit register address plus an offset. @@ -924,14 +1094,14 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { (u64{1} << 48)); break; } - case 0xb97: { // SULD.P.2D/2D_ARRAY.IGN, direct descriptor, channel mask at 72-75. + case 0xb97: { // SULD.P.2D/2D_ARRAY/3D.IGN, direct descriptor, channel mask at 72-75. const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); const u32 coordinate = insn.Bits(24, 8); const u32 channels = insn.Bits(72, 4); - const u32 dimension = insn.Bits(61, 3); // Maxwell's codes: 3 = 2D, 4 = 2D_ARRAY - if ((insn.words[2] & ~0xf00U) != 0x001e0000 || channels == 0 || (dimension != 3 && dimension != 4) || + const u32 dimension = insn.Bits(61, 3); // Maxwell's codes: 3 = 2D, 4 = 2D_ARRAY, 5 = 3D + if ((insn.words[2] & ~0xf00U) != 0x001e0000 || channels == 0 || dimension < 3 || dimension > 5 || binding >= 18 || offset > 0x1fff || insn.Bits(59, 2) || dest == 255 || - coordinate > (dimension == 4 ? 252U : 253U)) + coordinate > (dimension >= 4 ? 252U : 253U)) return fail(index, "native formatted surface load mode"); if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) return fail(index, "native surface descriptor buffer conflict"); @@ -944,14 +1114,35 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); const u32 data = insn.Bits(32, 8), coordinate = insn.Bits(24, 8); const u32 dimension = insn.Bits(61, 3); - if (insn.words[2] != 0x00100f00 || dimension < 3 || dimension > 5 || + const bool strong = insn.words[2] == 0x0010af00; // .STRONG.GPU: fenced on both sides + if ((insn.words[2] != 0x00100f00 && !strong) || dimension < 3 || dimension > 5 || binding >= 18 || offset > 0x1fff || insn.Bits(59, 2) || data > 252 || coordinate > (dimension >= 4 ? 252U : 253U)) return fail(index, "native formatted surface store mode"); + // A second descriptor buffer: SUST.B with the handle, loaded from that buffer, in R250. + const bool bindless = !bind_texture_buffer(binding); + if (bindless) { + emit(0x4c98000000070000ULL | 250 | (u64{15} << 39) | (u64{offset} << 20) | (u64{binding} << 34)); + } + if (strong) emit(0xef98000000000000ULL | predicate | (u64{1} << 8)); // MEMBAR.GL + emit(0xeb20000000000000ULL | predicate | data | (u64{coordinate} << 8) | + (u64{15} << 20) | (u64{dimension} << 33) | + (bindless ? u64{250} << 39 : (u64{offset} << 36) | (u64{1} << 51))); + if (strong) emit(0xef98000000000000ULL | predicate | (u64{1} << 8)); + break; + } + case 0x59f: { // SURED.D.2D.ADD.STRONG.GPU.IGN U32, direct descriptor (title 1's race compute). + const u32 binding = insn.Bits(54, 5), offset = insn.Bits(40, 14); + const u32 data = insn.Bits(32, 8), coordinate = insn.Bits(24, 8); + if (insn.words[2] != 0x0010e000 || insn.Bits(61, 3) != 3 || binding >= 18 || + offset > 0x1fff || insn.Bits(59, 2) || coordinate > 253) + return fail(index, "native surface reduction mode"); if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) return fail(index, "native surface descriptor buffer conflict"); result.required_texture_buffer = binding; - emit(0xeb20000000000000ULL | predicate | data | (u64{coordinate} << 8) | - (u64{15} << 20) | (u64{dimension} << 33) | (u64{offset} << 36) | (u64{1} << 51)); + // Maxwell SURED (as the frontend decodes it): operand 0-7, coordinates 8-15, ADD and + // U32 are zero at 24 and 20, type 33 (3 = 2D), bound offset 36, bound 51. + emit(0xeb40000000000000ULL | predicate | data | (u64{coordinate} << 8) | + (u64{3} << 33) | (u64{offset} << 36) | (u64{1} << 51)); break; } case 0x301: // BREV register. @@ -959,21 +1150,27 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(base(0x3828000000000000ULL) | (u64{insn.words[1]} << 8) | (u64{1} << 39) | (u64{1} << 40)); break; - case 0x219: case 0xc19: { // U32 register (or uniform, 0xc19) shifts with a zero word outside the selected result. + case 0x219: case 0xa19: case 0xc19: { // U32 register (or cbuf, 0xa19, or uniform, 0xc19) shifts with a zero word outside the selected result. const u32 low = insn.Bits(24, 8), high = insn.Bits(64, 8); - const u32 shift = opcode == 0xc19 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); + const bool constant = opcode == 0xa19; + // A constant shift amount is loaded into R251 once the form is known. + const u32 shift = opcode == 0xc19 ? uniform_register(insn.Bits(32, 8)) : constant ? 251U : insn.Bits(32, 8); + const bool shift_operand = constant ? valid_cbuf() && !insn.Bits(62, 2) : insn.words[1] <= 255; + const auto load_shift = [&] { if (constant) materialize(2, 0, 251, false, false); }; const bool right = insn.Bits(76, 1) != 0, high_result = insn.Bits(80, 1) != 0; // SHF.R.U32 of the low word with a zero high word: a plain logical right shift. if (right && !high_result && high == 255 && insn.Bits(73, 2) == 3 && shift != ~0U && - insn.words[1] <= 255 && + shift_operand && (insn.words[2] & ~(0x000118ffU | (opcode == 0xc19 ? 0x08000000U : 0U))) == 0x00000600U) { + load_shift(); emit(base(0x5c28000000000000ULL) | (u64{low} << 8) | (u64{shift} << 20)); break; } - if (insn.words[1] > 255 || insn.Bits(73, 2) != 3 || shift == ~0U || + if (!shift_operand || insn.Bits(73, 2) != 3 || shift == ~0U || (insn.words[2] & ~(0x000118ffU | (opcode == 0xc19 ? 0x08000000U : 0U))) != 0x00000600U || (right ? (!high_result || low != 255) : (high_result || high != 255))) return fail(index, "native register shift mode"); + load_shift(); emit(base(right ? 0x5cf8000000000000ULL : 0x5bf8000000000000ULL) | (u64{right ? high : 255U} << 8) | (u64{shift} << 20) | (u64{right ? 255U : low} << 39) | (u64{insn.Bits(75, 1)} << 50)); @@ -1012,8 +1209,8 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { if (insn.words[1] || insn.words[2]) return fail(index, "native error barrier mode"); emit(0xef98000000000000ULL | predicate | (u64{1} << 8)); break; - case 0xb1d: // BAR.SYNC 0, full workgroup. - if (insn.words[1] || insn.words[2] || native_predicate != 7) + case 0xb1d: // BAR.SYNC 0 (bit 80: .DEFER_BLOCKING), full workgroup. + if (insn.words[1] || (insn.words[2] & ~0x10000U) || native_predicate != 7) return fail(index, "native workgroup barrier mode"); emit(0xf0a8000000000000ULL | predicate | (u64{1} << 43) | (u64{1} << 44) | (u64{1} << 39) | (u64{7} << 39)); @@ -1188,10 +1385,21 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } break; } - case 0x919: case 0x9c3: { // S2R / S2UR thread and workgroup identities. + case 0x919: case 0x9c3: { // S2R / S2UR: special registers the Maxwell S2R handles. + // Same numbering as Maxwell: lane id (0), invocation id/info, Y direction, thread kill + // (17-19, 29), W scale factors (30-31, Kirby Air Riders), thread and workgroup ids, + // lane masks (56-60). const u32 special = insn.Bits(72, 8); + if (special == 132 && !insn.words[1] && insn.words[2] == (special << 8)) { + // SR_VARIABLE_RATE: without variable-rate shading every fragment is 1x1, its width + // and height in bytes 1 and 2 (Nintendo Switch Sports Resort multiplies them). + const u32 target = opcode == 0x9c3 ? uniform_register(static_cast<u32>(dest)) : static_cast<u32>(dest); + if (target == ~0U) return fail(index, "native uniform special destination"); + mov_imm(target, 0x00010100); + break; + } if (insn.words[1] || insn.words[2] != (special << 8) || - (special != 0 && special != 33 && special != 34 && special != 35 && + (special != 0 && (special < 17 || special > 19) && (special < 29 || special > 35) && special != 37 && special != 38 && special != 39 && special != 56 && special != 57 && special != 58 && special != 59 && special != 60)) return fail(index, "native special register"); @@ -1223,10 +1431,25 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { if (condition > 6 || target == ~0U || a == ~0U || b == ~0U || insn.Bits(12, 4) != 7 || (insn.words[2] & ~0x07800000U) != 0x08000000U) return fail(index, "native uniform select mode"); - if (opcode == 0x887 && index > 0 && condition == 0 && insn.words[1] <= 0x7ffff && - code[index - 1].Bits(0, 12) == 0x88c && - code[index - 1].Bits(24, 8) == insn.Bits(24, 8) && - code[index - 1].words[1] == insn.words[1] && code[index - 1].words[2] == 0x0bf01070 && + // The clamp's UISETP may sit a few instructions back, with only instructions between + // that write no uniform state and do not branch (Nintendo Switch Sports' jump tables), + // and no branch seen so far landing between them. + u32 compare = index; + for (u32 back = 1; back <= 8 && back <= index; ++back) { + const u32 earlier = code[index - back].Bits(0, 12), datapath = earlier & 0x1ff; + if (earlier == 0x88c) { + compare = index - back; + break; + } + if ((datapath >= 0x080 && datapath <= 0x0bf) || (datapath >= 0x1c2 && datapath <= 0x1c4) || + (earlier >= 0x940 && earlier <= 0x95f)) + break; + } + const bool target_between = std::any_of(relocations.begin(), relocations.end(), + [&](const Relocation& relocation) { return relocation.target > compare && relocation.target <= index; }); + if (opcode == 0x887 && compare < index && !target_between && condition == 0 && insn.words[1] <= 0x7ffff && + code[compare].Bits(12, 4) == 7 && code[compare].Bits(24, 8) == insn.Bits(24, 8) && + code[compare].words[1] == insn.words[1] && code[compare].words[2] == 0x0bf01070 && insn.words[2] == 0x08000000) { // Preserve the unsigned clamp idiom for indirect-table analysis. emit(0x3820000000070000ULL | target | (u64{a} << 8) | @@ -1257,10 +1480,17 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { // The uniform base sits where a register source would (bits 32-37). const u32 uniform_address = insn.Bits(38, 2) ? ~0U : uniform_register(insn.Bits(32, 6)); - if ((insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U | 0x2000U)) != 0x081e8100 || + // desc[URn][Ra.64]: the uniform pair is a memory (cache policy) descriptor and + // Ra the whole address. + const bool descriptor_address = (insn.words[2] & ~(0x3fU | 0xe00U)) == 0x0c1e1100; + if ((!descriptor_address && + (insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U | 0x2000U)) != 0x081e8100) || uniform_address == ~0U || (uniform_address & 1)) return fail(index, "native global load uniform addressing mode"); - if (address == 255) { + if (descriptor_address) { + if ((address & 1) || address >= 246) + return fail(index, "native global load register address pair"); + } else if (address == 255) { address = uniform_address; } else { if (insn.Bits(90, 1) && ((address & 1) || address >= 246)) @@ -1293,10 +1523,16 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { bool extended = insn.Bits(72, 1) != 0; if (opcode == 0x986) { const u32 uniform_address = uniform_register(insn.Bits(64, 6)); - if ((insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U)) != 0x0810a100 || + // desc[URn][Ra.64]: the uniform pair is a memory (cache policy) descriptor and + // Ra the whole address. + const bool descriptor_address = (insn.words[2] & ~(0x3fU | 0xe00U)) == 0x0c101100; + if ((!descriptor_address && (insn.words[2] & ~(0x04000000U | 0x3fU | 0xe00U)) != 0x0810a100) || uniform_address == ~0U || (uniform_address & 1) || !extended) return fail(index, "native global store uniform addressing mode"); - if (address == 255) address = uniform_address; + if (descriptor_address) { + if ((address & 1) || address >= 246) + return fail(index, "native global store register address pair"); + } else if (address == 255) address = uniform_address; else { if (insn.Bits(90, 1) && ((address & 1) || address >= 254)) return fail(index, "native global store register address pair"); @@ -1319,13 +1555,29 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { return fail(index, "native fragment discard mode"); emit(0xe33000000000000fULL | predicate); break; + case 0x311: { // F2I.FTZ.U64.TRUNC, register source, destination pair + if (insn.words[2] != 0x0021d800 || (insn.words[1] & ~0xffU) || insn.Bits(24, 8) || + (dest & 1) || dest >= 246) + return fail(index, "native unsigned 64-bit float conversion mode"); + emit(base(0x5cb0000000000000ULL) | (u64{3} << 8) | (u64{2} << 10) | + (u64{insn.Bits(32, 8)} << 20) | (u64{3} << 39) | (u64{1} << 44)); + break; + } + case 0x312: { // I2F.F32.(U|S)64, register pair source; sign (74) and rounding (78-79) as Maxwell's + if ((insn.words[2] & ~0xc400U) != 0x00301000 || (insn.words[1] & ~0xffU) || insn.Bits(24, 8) || + (insn.Bits(32, 8) & 1) || insn.Bits(32, 8) >= 254) + return fail(index, "native 64-bit integer-to-float mode"); + emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{3} << 10) | (u64{insn.Bits(74, 1)} << 13) | + (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(78, 2)} << 39)); + break; + } case 0x305: case 0xb05: case 0xd05: { // F2I NTZ, register/constant-buffer/uniform source const u32 mode = insn.words[2] & ~(opcode == 0xd05 ? 0x08000000U : 0U); const bool half = opcode == 0x305 && mode == 0x0010f000; const bool small = mode == 0x00212800; const bool floor = (mode & ~0x10100U) == 0x00207000; const bool nearest = (mode & ~0x10100U) == 0x00203000; - if ((!half && !small && !floor && !nearest && (mode & ~0x100U) != 0x0021f000) || + if ((!half && !small && !floor && !nearest && (mode & ~0x10100U) != 0x0020f000) || (opcode != 0xb05 && (insn.words[1] & 0xffffff00U)) || (opcode == 0xd05 && uniform_register(insn.Bits(32, 8)) == ~0U)) { return fail(index, "native float-to-integer conversion mode"); @@ -1363,19 +1615,42 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } break; } - case 0xb6f: { // TXQ texture dimensions, two packed scalar outputs. - if (insn.Bits(59, 5) || insn.Bits(64, 8) != 255 || insn.Bits(72, 4) == 0 || insn.Bits(72, 4) > 3 || - insn.Bits(76, 14) || insn.Bits(91, 1) || insn.Bits(54, 5) != 2 || - insn.Bits(40, 14) > 0x1fff || (insn.Bits(72, 4) == 3 && (dest & 1)) || dest >= 254) + case 0xb6f: case 0x370: { // TXQ texture dimensions (0x370: TXQ.B). + // TXQ.B takes the handle in Ra and the level in Ra+1, as Maxwell's. Up to two outputs + // go to Rd, Rd+1 directly; more, or a second destination (bits 64-71) for the third and + // fourth, come out packed in scratch and are copied as for TEX. + const bool bindless = opcode == 0x370; + const u32 binding = insn.Bits(54, 5); // descriptor buffer, as for TLD + const u32 mask = insn.Bits(72, 4), components = static_cast<u32>(std::popcount(mask)); + const u32 second_dest = insn.Bits(64, 8); + const bool direct = second_dest == 255; + if (insn.Bits(59, 5) != (bindless ? 1U : 0U) || mask == 0 || (direct && components > 2) || + insn.Bits(76, 14) || insn.Bits(91, 1) || (!bindless && binding >= 18) || + (bindless && (insn.Bits(32, 27) || insn.Bits(24, 8) >= 245)) || + (!direct && (second_dest >= 245 || dest >= 245)) || + insn.Bits(40, 14) > 0x1fff || (direct && components == 2 && (dest & 1)) || dest >= 254) return fail(index, "native texture dimension query mode"); - result.required_texture_buffer = 2; - emit(0xdf48000000000000ULL | predicate | dest | (u64{insn.Bits(24, 8)} << 8) | - (u64{1} << 22) | (u64{insn.Bits(72, 4)} << 31) | (u64{insn.Bits(40, 14)} << 36) | - (u64{insn.Bits(90, 1)} << 49)); + if (!bindless) { + if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) + return fail(index, "native texture descriptor buffer conflict"); + result.required_texture_buffer = binding; + } + emit((bindless ? 0xdf50000000000000ULL : 0xdf48000000000000ULL) | predicate | (direct ? dest : 250) | + (u64{insn.Bits(24, 8)} << 8) | (u64{1} << 22) | (u64{mask} << 31) | + (u64{bindless ? 0U : insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49)); + for (u32 i = 0; !direct && i < components; ++i) { + const u32 target = i < 2 ? (dest == 255 ? 255U : static_cast<u32>(dest) + i) : second_dest + (i - 2); + if (target != 255) + emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{250 + i} << 20)); + } break; } - case 0xb66: { // TLD.SCR.LZ, direct texel fetch + case 0xb66: { // TLD.SCR, direct texel fetch at level zero (.LZ) or a given level (.LL) const u32 mask = insn.Bits(72, 4); + const bool level = insn.Bits(87, 3) == 3; // .LL: the level follows the coordinates + // Bit 76: a texel offset after the coordinates and level. 2D only, as Maxwell TLDS + // 2D.LZ.AOFFI (x, y in Ra, Ra+1, offsets in Rb) or 2D.LL.AOFFI (level, offsets in Rb, Rb+1). + const bool offsets = insn.Bits(76, 1) != 0; const u32 dimension = insn.Bits(61, 3); const bool array = dimension == 5; const bool volume = dimension == 2; @@ -1391,8 +1666,12 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { for (u32 i = 0; i < rg_masks.size(); ++i) if (rg_masks[i] == mask) swizzle = i; } if (!insn.Bits(60, 1) || (dimension != 0 && dimension != 1 && !array && !volume) || - swizzle == ~0U || insn.Bits(76, 5) || - insn.Bits(81, 3) != 7 || insn.Bits(87, 3) != 1 || + swizzle == ~0U || (insn.Bits(76, 5) & ~1U) || + (offsets && (dimension != 1 || (source_a & 1) || source_a >= 254 || source_b == 255 || + (level && ((source_b & 1) || source_b >= 254)))) || + insn.Bits(81, 3) != 7 || (insn.Bits(87, 3) != 1 && !level) || (level && volume) || + (level && dimension == 1 && ((source_a & 1) || source_a >= 254)) || + (level && array && ((source_b & 1) || source_b >= 254)) || insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18 || dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) || @@ -1402,9 +1681,46 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { return fail(index, "native texel fetch mode or descriptor binding"); } const u32 binding = insn.Bits(54, 5); - if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) - return fail(index, "native texture descriptor buffer conflict"); - result.required_texture_buffer = binding; + if (!bind_texture_buffer(binding)) { + if (level || offsets) + return fail(index, "native texture descriptor buffer conflict"); + // A second descriptor buffer: TLD.B (LZ) with the coordinates in R250.. (array + // index or Z after Ra) and the handle, loaded from that buffer, in R253. + const auto copy = [&](u32 target, u32 source) { + emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{source} << 20)); + }; + copy(250, source_a); + if (array || volume) copy(251, source_a + 1); + if (dimension != 0) copy(array || volume ? 252 : 251, source_b); + emit(0x4c98000000070000ULL | 253 | (u64{15} << 39) | (u64{insn.Bits(40, 14)} << 20) | + (u64{binding} << 34)); + emit(0xdd00000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{253} << 20) | + (u64{array ? 3U : volume ? 4U : dimension == 0 ? 0U : 2U} << 28) | (u64{mask} << 31) | + (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51)); + const u32 components = static_cast<u32>(std::popcount(mask)); + for (u32 i = 0; i < components; ++i) + copy(i < 2 || !split ? static_cast<u32>(dest) + i : second_dest + (i - 2), 250 + i); + break; + } + if (array && level) { + // TLDS has no array form with a level: TLD takes (layer, x, y) from Ra and the level + // from Rb (layer, x in Ra, Ra+1; y, level in Rb, Rb+1 here). Results come out packed, + // then go where the Ampere destinations (Rd, Rd+1, then the second pair) take them. + for (u32 i = 0; i < 2; ++i) { + emit(0x5c98000000000000ULL | predicate | (250 + i) | (u64{15} << 39) | (u64{source_a + i} << 20)); + emit(0x5c98000000000000ULL | predicate | (252 + i) | (u64{15} << 39) | (u64{source_b + i} << 20)); + } + emit(0xdc00000000000000ULL | predicate | 246 | (u64{250} << 8) | (u64{253} << 20) | + (u64{3} << 28) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | + (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51) | (u64{1} << 55)); + const u32 components = static_cast<u32>(std::popcount(mask)); + for (u32 i = 0; i < components; ++i) { + const u32 target = !split || i < 2 ? static_cast<u32>(dest) + i : second_dest + (i - 2); + if (target != 255 && (split || i < 2)) + emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{246 + i} << 20)); + } + break; + } if (array) { emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a + 1} << 20)); emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | (u64{source_b} << 20)); @@ -1413,7 +1729,42 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(0xda00000000000000ULL | predicate | dest | (u64{source_a} << 8) | (u64{source_b} << 20) | (u64{insn.Bits(64, 8)} << 28) | (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | - (u64{swizzle} << 50) | (u64{array ? 8U : volume ? 7U : dimension == 0 ? 0U : 2U} << 53)); + (u64{swizzle} << 50) | + (u64{offsets ? (level ? 12U : 4U) + : array ? 8U : volume ? 7U : dimension == 0 ? (level ? 1U : 0U) : (level ? 5U : 2U)} << 53)); + break; + } + case 0x367: { // TLD.B at level zero (.LZ), the handle in a register. + // The values are the coordinates, then the handle, split over Ra and Rb as for TEX.B + // (ARRAY_2D: layer, x | y, handle); Maxwell TLD.B takes the handle first in Rb. + constexpr std::array<u32, 8> coordinate_count{1, 2, 3, 0, 0, 3, 0, 0}; + constexpr std::array<u32, 8> maxwell_type{0, 2, 4, 0, 0, 3, 0, 0}; + const u32 mask = insn.Bits(72, 4), second_dest = insn.Bits(64, 8); + const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); + const u32 dimension = insn.Bits(61, 3), coordinates = coordinate_count[dimension]; + const u32 count = coordinates + 1, in_a = (count + 1) / 2; + if (coordinates == 0 || !insn.Bits(59, 1) || !insn.Bits(60, 1) || insn.Bits(40, 19) || + insn.Bits(76, 5) || insn.Bits(81, 3) != 7 || insn.Bits(87, 3) != 1 || insn.Bits(91, 1) || + mask == 0 || source_a == 255 || source_a + in_a > 246 || source_b == 255 || + source_b + (count - in_a) > 246) + return fail(index, "native bindless texel fetch mode"); + const auto value = [&](u32 i) { return i < in_a ? source_a + i : source_b + (i - in_a); }; + const auto copy = [&](u32 target, u32 source) { + emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{source} << 20)); + }; + for (u32 i = 0; i < coordinates; ++i) copy(250 + i, value(i)); + copy(246, value(coordinates)); + emit(0xdd00000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{246} << 20) | + (u64{maxwell_type[dimension]} << 28) | (u64{mask} << 31) | (u64{insn.Bits(90, 1)} << 49) | + (u64{7} << 51)); + const u32 components = static_cast<u32>(std::popcount(mask)); + const bool split = second_dest != 255; + for (u32 i = 0; i < components; ++i) { + u32 target = 255; + if (!split || i < 2) target = dest == 255 ? 255U : static_cast<u32>(dest) + i; + else target = second_dest + (i - 2); + if (target != 255) copy(target, 250 + i); + } break; } case 0x91a: // DEPBAR.LE SB5, 1: scoreboard wait, represented by IR data dependencies. @@ -1421,22 +1772,35 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { return fail(index, "native dependency barrier mode"); } break; - case 0xb69: { // TMML (LOD query), 2D. Results are 8.8 fixed point, as on Maxwell. + case 0xb69: case 0xf69: case 0x36a: { // TMML (LOD query), 1D/2D/3D/CUBE. Results are 8.8 fixed point, as on Maxwell. + // 0xf69: the handle in a uniform register (bit 91); 0x36a, TMML.B: the handle in Rb and + // every coordinate in Ra. Both become Maxwell TMML.B with the handle in Rb. + const bool bindless = opcode != 0xb69; + const u32 handle = opcode == 0xf69 ? uniform_register(insn.Bits(40, 6)) + : opcode == 0x36a ? insn.Bits(32, 8) : 255; const u32 mask = insn.Bits(72, 4), second_dest = insn.Bits(64, 8); const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); - const u32 binding = insn.Bits(54, 5); - if (insn.Bits(61, 3) != 1 || insn.Bits(76, 4) || insn.Bits(80, 10) || insn.Bits(59, 1) || - insn.Bits(40, 14) > 0x1fff || binding >= 18 || mask == 0 || source_a >= 246 || - (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)) + const u32 binding = insn.Bits(54, 5), dimension = insn.Bits(61, 3); + // Coordinates: all from Ra when Rb is RZ, else split as for TEX (2D: Ra, Rb). + constexpr std::array<u32, 4> coordinate_count{1, 2, 3, 3}; + const u32 coordinates = dimension < 4 ? coordinate_count[dimension] : 0; + const u32 in_a = source_b == 255 || opcode == 0x36a ? coordinates : (coordinates + 1) / 2; + if (coordinates == 0 || insn.Bits(76, 4) || insn.Bits(80, 10) || mask == 0 || + insn.Bits(59, 1) != (opcode == 0x36a ? 1U : 0U) || source_a + in_a > 246 || + (in_a < coordinates && source_b + (coordinates - in_a) > 246) || + (opcode == 0xf69 && (handle >= 246 || insn.Bits(46, 13) || !insn.Bits(91, 1))) || + (opcode == 0x36a && (handle >= 246 || insn.Bits(40, 19) || insn.Bits(91, 1))) || + (!bindless && (insn.Bits(40, 14) > 0x1fff || binding >= 18 || + (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)))) return fail(index, "native texture LOD query mode"); - result.required_texture_buffer = binding; - // Coordinates: (Ra, Ra+1) when Rb is RZ, else (Ra, Rb). - emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a} << 20)); - emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | - (u64{source_b == 255 ? source_a + 1 : source_b} << 20)); - emit(0xdf58000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | - (u64{2} << 28) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | - (u64{insn.Bits(90, 1)} << 49)); + if (!bindless) result.required_texture_buffer = binding; + for (u32 i = 0; i < coordinates; ++i) + emit(0x5c98000000000000ULL | predicate | (250 + i) | (u64{15} << 39) | + (u64{i < in_a ? source_a + i : source_b + (i - in_a)} << 20)); + if (bindless) emit(0x5c98000000000000ULL | predicate | 246 | (u64{15} << 39) | (u64{handle} << 20)); + emit((bindless ? 0xdf60000000000000ULL : 0xdf58000000000000ULL) | predicate | 250 | (u64{250} << 8) | + (u64{bindless ? 246U : 255U} << 20) | (u64{dimension * 2} << 28) | (u64{mask} << 31) | + (u64{bindless ? 0U : insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49)); const u32 components = static_cast<u32>(std::popcount(mask)); const bool split = second_dest != 255; for (u32 i = 0; i < components; ++i) { @@ -1448,22 +1812,43 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } break; } - case 0xb63: { // TLD4 (gather), 2D without offsets or depth compare; component at bits 87-88. + case 0xb63: case 0xf63: { // TLD4 (gather), 2D without offsets or depth compare; component at bits 87-88. + // 0xf63: the handle in a uniform register (bit 91), Maxwell TLD4.B with the handle in Rb. + const bool bindless = opcode == 0xf63; + const u32 handle = bindless ? uniform_register(insn.Bits(40, 6)) : 255; const u32 mask = insn.Bits(72, 4), second_dest = insn.Bits(64, 8); const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); const u32 binding = insn.Bits(54, 5); - if (insn.Bits(61, 3) != 1 || insn.Bits(76, 4) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || - insn.Bits(40, 14) > 0x1fff || binding >= 18 || insn.Bits(89, 1) || mask == 0 || - (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)) + // Bit 76: a texel offset after the coordinates (x, y in Ra, Ra+1, offsets in Rb, as + // for TEX); Maxwell TLD4 takes it as AOFFI (bits 54-55) from Rb. + const bool offsets = insn.Bits(76, 1) != 0; + if (insn.Bits(61, 3) != 1 || insn.Bits(77, 3) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || + (offsets && (bindless || source_b == 255 || (source_a != 255 && source_a >= 245))) || + insn.Bits(89, 1) || mask == 0 || + (bindless ? (handle >= 246 || insn.Bits(46, 13) || !insn.Bits(91, 1)) + : (insn.Bits(40, 14) > 0x1fff || binding >= 18 || + (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding)))) return fail(index, "native texture gather mode"); - result.required_texture_buffer = binding; - // Two coordinates: x in Ra, y in Rb (the coordinate list splits evenly across them). + if (!bindless) result.required_texture_buffer = binding; + // Two coordinates: x in Ra, y in Rb (the coordinate list splits evenly across them); + // with an offset, x and y in Ra, Ra+1 and the offset in Rb. + const u32 y = !offsets ? source_b : source_a == 255 ? 255U : source_a + 1; emit(0x5c98000000000000ULL | predicate | 250 | (u64{15} << 39) | (u64{source_a} << 20)); - emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | - (u64{source_b == 255 ? 255U : source_b} << 20)); - emit(0xc800000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | - (u64{2} << 28) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | - (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51) | (u64{insn.Bits(87, 2)} << 56)); + emit(0x5c98000000000000ULL | predicate | 251 | (u64{15} << 39) | (u64{y} << 20)); + if (offsets) { + emit(0x5c98000000000000ULL | predicate | 246 | (u64{15} << 39) | (u64{source_b} << 20)); + } + if (bindless) { + emit(0x5c98000000000000ULL | predicate | 246 | (u64{15} << 39) | (u64{handle} << 20)); + emit(0xdec0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{246} << 20) | + (u64{2} << 28) | (u64{mask} << 31) | (u64{insn.Bits(87, 2)} << 38) | + (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51)); + } else { + emit(0xc800000000000000ULL | predicate | 250 | (u64{250} << 8) | + (u64{offsets ? 246U : 255U} << 20) | (u64{2} << 28) | (u64{mask} << 31) | + (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | (u64{7} << 51) | + (u64{offsets ? 1U : 0U} << 54) | (u64{insn.Bits(87, 2)} << 56)); + } const u32 components = static_cast<u32>(std::popcount(mask)); const bool split = second_dest != 255; for (u32 i = 0; i < components; ++i) { @@ -1475,7 +1860,15 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } break; } - case 0xb60: { // TEX, scalar 2D implicit-LOD sample + case 0xb60: case 0xf60: case 0x361: { // TEX; 0xf60: handle in a uniform register; 0x361: TEX.B + // Bindless forms become Maxwell TEX.B, the handle first among the meta registers (the + // texture pass tracks it back to its constant buffer words). 0x361 lists the handle + // right after the coordinates, then the LOD and the offsets (see general_tex). + const bool bindless = opcode != 0xb60; + const u32 handle = opcode == 0xf60 ? uniform_register(insn.Bits(40, 6)) : 255; + if (bindless && ((opcode == 0xf60 && (handle == ~0U || handle >= 246 || insn.Bits(46, 13))) || + (opcode == 0x361 && (insn.Bits(40, 19) || insn.Bits(87, 3) > 3)))) + return fail(index, "native bindless texture mode"); const u32 mask = insn.Bits(72, 4); const u32 second_dest = insn.Bits(64, 8); constexpr std::array<u32, 8> rg_masks{1, 2, 4, 8, 3, 9, 10, 12}; @@ -1485,12 +1878,12 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { const bool depth_compare = insn.Bits(78, 1); const u32 dimension = insn.Bits(61, 3), lod = insn.Bits(87, 3); const u32 source_a = insn.Bits(24, 8), source_b = insn.Bits(32, 8); - if (dimension == 5 && depth_compare && lod == 0 && !split && mask == 1) { + if (!bindless && dimension == 5 && depth_compare && lod == 0 && !split && mask == 1) { if (!insn.Bits(60, 1) || insn.Bits(76, 2) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) != 2 || (source_a & 1) || (source_b & 1) || source_a >= 248 || source_b >= 248 || dest >= 248) return fail(index, "native array depth comparison sample mode"); - result.required_texture_buffer = 2; + if (!bind_texture_buffer(2)) return fail(index, "native texture descriptor buffer conflict"); for (u32 i = 0; i < 3; ++i) { const u32 source = i < 2 ? source_a + i : source_b; emit(0x5c98000000000000ULL | predicate | (250 + i) | @@ -1510,14 +1903,23 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { constexpr std::array<u32, 8> coordinate_count{1, 2, 3, 3, 2, 3, 0, 4}; constexpr std::array<u32, 8> maxwell_type{0, 2, 4, 6, 1, 3, 0, 7}; const bool offsets = insn.Bits(76, 1) != 0; - if (dimension == 6 || insn.Bits(77, 1) || insn.Bits(81, 3) != 7 || insn.Bits(59, 1) || - insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18 || (lod > 3 && lod != 6 && lod != 7) || - (depth_compare && mask != 1) || mask == 0) + // Bit 59 is .B, set in 0x361's encoding. + if (dimension == 6 || insn.Bits(77, 1) || insn.Bits(81, 3) != 7 || + insn.Bits(59, 1) != (opcode == 0x361 ? 1U : 0U) || + (!bindless && (insn.Bits(40, 14) > 0x1fff || insn.Bits(54, 5) >= 18)) || + (lod > 3 && lod != 6 && lod != 7) || (depth_compare && mask != 1) || mask == 0) return false; const u32 coordinates = coordinate_count[dimension]; const bool has_lod = lod == 2 || lod == 3 || lod == 6 || lod == 7; - const u32 count = coordinates + (has_lod ? 1 : 0) + (offsets ? 1 : 0) + (depth_compare ? 1 : 0); - const u32 in_a = count <= 4 ? (count + 1) / 2 : 4; + // 0x361's handle is one more value after the coordinates: 2D.DC is (x, y | handle, + // reference), ARRAY_2D is (layer, x | y, handle); with five or more values the + // coordinates keep Ra (CUBE.LL is (x, y, z | handle, LOD)). + const u32 handle_slot = opcode == 0x361 ? 1 : 0; + const u32 count = coordinates + handle_slot + (has_lod ? 1 : 0) + (offsets ? 1 : 0) + + (depth_compare ? 1 : 0); + // Unpacked 0x361 (bit 60 clear): Ra holds the coordinates, Rb the handle and the rest. + const u32 in_a = opcode == 0x361 && !insn.Bits(60, 1) ? coordinates + : count <= 4 ? (count + 1) / 2 : opcode == 0x361 ? coordinates : 4; const auto value = [&](u32 i) -> u32 { if (i < in_a) return source_a == 255 ? 255U : source_a + i; return source_b == 255 ? 255U : source_b + (i - in_a); @@ -1525,25 +1927,40 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { if ((source_a != 255 && source_a + in_a > 246) || (count > in_a && source_b != 255 && source_b + (count - in_a) > 246)) return false; + const u32 texture_handle = opcode == 0x361 ? value(coordinates) : handle; + if (bindless && texture_handle >= 246) return false; + // A second descriptor buffer is read bindless: TEX.B takes the handle, loaded from + // that buffer, as its first meta register. const u32 binding = insn.Bits(54, 5); - if (result.required_texture_buffer != ~0U && result.required_texture_buffer != binding) - return false; - result.required_texture_buffer = binding; + const bool second_buffer = !bindless && !bind_texture_buffer(binding); + const bool tex_b = bindless || second_buffer; const auto copy = [&](u32 target, u32 source) { emit(0x5c98000000000000ULL | predicate | target | (u64{15} << 39) | (u64{source} << 20)); }; // Coordinates to R250.., then LOD/bias, offsets and reference to R246... for (u32 i = 0; i < coordinates; ++i) copy(250 + i, value(i)); u32 meta = 246; - u32 next = coordinates; + u32 next = coordinates + handle_slot; + if (bindless) copy(meta++, texture_handle); + else if (second_buffer) + emit(0x4c98000000070000ULL | meta++ | (u64{15} << 39) | (u64{insn.Bits(40, 14)} << 20) | + (u64{binding} << 34)); if (has_lod) copy(meta++, value(next++)); if (offsets) copy(meta++, value(next++)); if (depth_compare) copy(meta++, value(next++)); + if (tex_b) { + // Maxwell TEX.B: AOFFI at 36 and the LOD mode at 37 instead of 54 and 55. + emit(0xde80000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{246} << 20) | + (u64{maxwell_type[dimension]} << 28) | (u64{mask} << 31) | (u64{offsets} << 36) | + (u64{lod} << 37) | (u64{insn.Bits(90, 1)} << 49) | (u64{depth_compare} << 50) | + (u64{7} << 51)); + } else { emit(0xc000000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{meta == 246 ? 255U : 246U} << 20) | (u64{maxwell_type[dimension]} << 28) | (u64{mask} << 31) | (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | (u64{depth_compare} << 50) | (u64{7} << 51) | (u64{offsets} << 54) | (u64{lod} << 55)); + } const u32 components = static_cast<u32>(std::popcount(mask)); if (insn.Bits(79, 1)) { // FP16 results: (c0, c1) packed into the first destination, (c2, c3) into @@ -1578,7 +1995,7 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { // SM70 scalar coordinates and split RG/BA destinations match TEXS. // Descriptor offsets are words in both encodings, but only native // explicitly encodes the constant-buffer binding. - if (!insn.Bits(60, 1) || (dimension != 1 && dimension != 5) || + if (bindless || !insn.Bits(60, 1) || (dimension != 1 && dimension != 5) || swizzle == ~0U || insn.Bits(76, 2) || insn.Bits(81, 3) != 7 || (lod > 1 && (lod != 3 || dimension != 1 || depth_compare)) || (lod == 3 && ((source_a & 1) || source_a >= 249)) || @@ -1587,11 +2004,11 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { dest == 255 || (mask & (mask - 1) && ((dest & 1) || dest >= 254)) || (split && mask == 15 && ((second_dest & 1) || second_dest >= 254)) || (depth_compare && (dimension != 1 || split || mask != 1 || (source_a & 1) || source_a >= 249)) || - (dimension == 5 && ((source_a & 1) || source_a >= 249))) { + (dimension == 5 && ((source_a & 1) || source_a >= 249)) || bindless || + !bind_texture_buffer(2)) { if (!general_tex()) return fail(index, "native texture mode or descriptor binding"); break; } - result.required_texture_buffer = 2; emit((0xd800000000000000ULL & ~(u64{insn.Bits(79, 1)} << 59)) | predicate | dest | (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(64, 8)} << 28) | (u64{insn.Bits(40, 14)} << 36) | (u64{insn.Bits(90, 1)} << 49) | @@ -1631,11 +2048,12 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } break; } - case 0x307: { // FRND.F32, round to an integral floating-point value + case 0x307: case 0xb07: { // FRND.F32, round to an integral floating-point value (0xb07: cbuf) if (insn.Bits(75, 2) != 2 || insn.Bits(84, 2) != 2 || insn.Bits(81, 3) || insn.Bits(86, 6)) return fail(index, "native floating round format"); - emit(base(0x5ca8000000000000ULL) | (u64{2} << 8) | (u64{2} << 10) | - (u64{insn.Bits(32, 8)} << 20) | (u64{8 + insn.Bits(78, 2)} << 39) | + if (opcode == 0xb07 && !valid_cbuf()) return fail(index, "native constant buffer address"); + emit(base(opcode == 0xb07 ? 0x4ca8000000000000ULL : 0x5ca8000000000000ULL) | (u64{2} << 8) | (u64{2} << 10) | + (opcode == 0xb07 ? cbuf() : u64{insn.Bits(32, 8)} << 20) | (u64{8 + insn.Bits(78, 2)} << 39) | (u64{insn.Bits(80, 1)} << 44) | (u64{insn.Bits(63, 1)} << 45) | (u64{insn.Bits(62, 1)} << 49)); break; @@ -1666,15 +2084,22 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { result.instructions.push_back(translated); break; } - case 0x387: case 0x983: { // Captured scalar local spill at an immediate byte offset. + case 0x387: case 0x983: { // Local spill: 32, 64 or 128 bits at [Ra + immediate byte offset]. + // The size is bits 73-75 (4, 5, 6 = 32, 64, 128 bits); wider accesses become 32-bit ones. + // A load's bit 85 (.LU) is a cache hint. const bool store = opcode == 0x387; - const u32 data = insn.Bits(32, 8), offset = insn.Bits(40, 24); - if (insn.Bits(24, 8) != 255 || (store && (insn.Bits(16, 8) || data > 247)) || - (!store && (data || dest >= 248)) || (offset & 3) || offset > 0x7fffff || - insn.words[2] != (store ? 0x00100800U : 0x00300800U)) + const u32 size = insn.Bits(73, 3); + const u32 data = insn.Bits(32, 8), offset = insn.Bits(40, 24), address = insn.Bits(24, 8); + const u32 words = size == 6 ? 4 : size == 5 ? 2 : 1, value = store ? data : static_cast<u32>(dest); + if ((address != 255 && address >= 246) || (store && insn.Bits(16, 8)) || (!store && data) || + (value != 255 && (value + words > 248 || (value & (words - 1)))) || (!store && value == 255) || + (offset & 3) || offset + 4 * words > 0x800000 || size < 4 || size > 6 || + (insn.words[2] & ~(store ? 0xe00U : 0x200e00U)) != 0x00100000U) return fail(index, "native local scalar spill mode"); - emit((store ? 0xef50000000000000ULL : 0xef40000000000000ULL) | predicate | - (store ? data : dest) | (u64{255} << 8) | (u64{offset} << 20) | (u64{4} << 48)); + for (u32 word = 0; word < words; ++word) + emit((store ? 0xef50000000000000ULL : 0xef40000000000000ULL) | predicate | + (value == 255 ? 255U : value + word) | (u64{address} << 8) | + (u64{offset + 4 * word} << 20) | (u64{4} << 48)); break; } case 0x803: { // P2R PR, register merge source, immediate mask, byte select (76-77) @@ -1739,17 +2164,30 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { break; } case 0x209: case 0x809: case 0xa09: case 0xc09: { // FMNMX, including a uniform source - if (insn.Bits(74, 6) || insn.Bits(81, 6) || (opcode != 0xc09 && insn.Bits(91, 1))) + if (insn.Bits(74, 6) || insn.Bits(82, 5) || (opcode != 0xc09 && insn.Bits(91, 1))) return fail(index, "native floating min/max mode"); const u32 a = operand(0), b = operand(1); + // .NAN (bit 81): an unordered pair gives the canonical NaN. R252 = ~0 when both are numbers. + const bool propagate_nan = insn.Bits(81, 1); + if (propagate_nan) + emit(0x5800000000000000ULL | predicate | 252 | (u64{a} << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{7} << 48)); emit(base(0x5c60000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42) | (u64{insn.Bits(80, 1)} << 44)); + if (propagate_nan) { + emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{252} << 20) | (u64{7} << 48)); + emit(0x0400000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{0xffffffffU} << 20) | (u64{2} << 53)); + emit(0x0400000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{0x7fffffffU} << 20)); + emit(base(0x5c40000000000000ULL) | (dest << 8) | (u64{252} << 20) | (u64{1} << 41) | (u64{7} << 48)); + } break; } - case 0x308: case 0xb08: { // MUFU + case 0x308: case 0xb08: case 0xd08: { // MUFU (register, constant or uniform register source) const u32 operation = insn.Bits(74, 6); - if (operation > 8 || operation == 6 || operation == 7 || insn.Bits(80, 12)) return fail(index, "native MUFU mode"); + // Bit 91 is part of the uniform-register form's encoding. + if (operation > 8 || operation == 6 || operation == 7 || + insn.Bits(80, 12) != (opcode == 0xd08 ? 0x800U : 0U)) + return fail(index, "native MUFU mode"); // SM70+ MUFU.COS/SIN take the angle in turns (compilers multiply by 1/2pi first); // Eden's Maxwell MUFU takes radians (its RRO is a plain move), so scale by 2pi. const bool turns = operation == 0 || operation == 1; @@ -1768,7 +2206,8 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(0x5080000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{operation} << 20)); emit(base(0x5ca8000000000000ULL) | (u64{1} << 8) | (u64{2} << 10) | (u64{250} << 20)); } else { - u32 src = operand(1); + u32 src = opcode == 0xd08 ? uniform_register(insn.Bits(32, 8)) : operand(1); + if (src == ~0U) return fail(index, "native MUFU uniform source"); if (turns) src = to_radians(src); emit(base(0x5080000000000000ULL) | (u64{src} << 8) | (u64{operation} << 20)); } @@ -1838,7 +2277,7 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { mov_imm(static_cast<u32>(dest) + 1, 0); break; } - case 0x306: case 0xd06: { // I2F.F32.U8, low byte, round-to-nearest (0xd06: uniform source) + case 0x306: case 0xd06: { // I2F.F32.U8, round-to-nearest (0xd06: uniform source) const u32 source = opcode == 0xd06 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); if (opcode == 0x306 && insn.words[2] == 0x00101000 && !(insn.words[1] & 0xcfffff00U)) { // I2F.U16 Rd, Rs(.H1): Maxwell's 16-bit source format, byte selector 0 or 2. @@ -1846,10 +2285,12 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { (u64{insn.Bits(60, 2) * 2} << 41)); break; } + // I2F.U8 Rd, Rs.B0-B3: the byte selector (bits 60-61) as Maxwell's (41-42). if (insn.words[2] != (opcode == 0xd06 ? 0x08001000U : 0x1000U) || - (insn.words[1] & 0xffffff00U) || source == ~0U) + (insn.words[1] & (opcode == 0xd06 ? 0xffffff00U : 0xcfffff00U)) || source == ~0U) return fail(index, "native byte-to-float conversion mode"); - emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{source} << 20)); + emit(base(0x5cb8000000000000ULL) | (u64{2} << 8) | (u64{source} << 20) | + (u64{insn.Bits(60, 2)} << 41)); break; } case 0x245: case 0xa45: case 0xc45: { // I2FP.F32.U32 @@ -1862,13 +2303,15 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } case 0x207: case 0x807: case 0xa07: case 0x208: case 0x808: case 0xa08: { // Bitwise SEL / FSEL - const u32 a = operand(0, false), b = operand(1, false); + // FSEL applies the floating operand modifiers (negate/absolute) before selecting. + const bool floating = (opcode & 0x1ff) == 0x008; + const u32 a = operand(0, floating), b = operand(1, floating); emit(base(0x5ca0000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); break; } - case 0x20a: case 0x80a: case 0xa0a: { // FSET.BF, combined with Pc (bits 87-90) by bits 74-75 - if (insn.Bits(64, 8) || insn.Bits(74, 2) == 3 || insn.Bits(81, 6) || insn.Bits(91, 1)) + case 0x20a: case 0x80a: case 0xa0a: case 0xc0a: { // FSET.BF, combined with Pc (bits 87-90) by bits 74-75 (0xc0a: uniform b) + if (insn.Bits(64, 8) || insn.Bits(74, 2) == 3 || insn.Bits(81, 6) || (opcode != 0xc0a && insn.Bits(91, 1))) return fail(index, "native floating comparison output mode"); const u32 a = operand(0), b = operand(1); emit(base(0x5800000000000000ULL) | (u64{a} << 8) | (u64{b} << 20) | @@ -1876,21 +2319,61 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { (u64{insn.Bits(76, 4)} << 48) | (u64{1} << 52) | (u64{insn.Bits(80, 1)} << 55)); break; } - case 0x434: { // HSETP2 with an immediate half pair: A swizzle 74-75 (neg 72, abs 73), + case 0x234: case 0x434: { // HSETP2 with a register (B swizzle 60-61, abs 62, neg 63) or an + // immediate half pair: A swizzle 74-75 (neg 72, abs 73), // combine 69-70, H_AND 71, compare 76-79, FTZ 80, P/Q 81-86, input 87-90. - mov_imm(250, insn.words[1]); + u64 source = u64{250} << 20; + if (opcode == 0x234) { + if ((insn.words[1] & 0x0fffff00U) || insn.Bits(64, 5)) + return fail(index, "native half comparison mode"); + source = (u64{insn.Bits(32, 8)} << 20) | (u64{insn.Bits(60, 2)} << 28) | + (u64{insn.Bits(62, 1)} << 30) | (u64{insn.Bits(63, 1)} << 31); + } else { + mov_imm(250, insn.words[1]); + } emit(0x5d20000000000000ULL | predicate | insn.Bits(84, 3) | (u64{insn.Bits(81, 3)} << 3) | - (u64{insn.Bits(80, 1)} << 6) | (u64{insn.Bits(24, 8)} << 8) | (u64{250} << 20) | + (u64{insn.Bits(80, 1)} << 6) | (u64{insn.Bits(24, 8)} << 8) | source | (u64{insn.Bits(76, 4)} << 35) | (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42) | (u64{insn.Bits(72, 1)} << 43) | (u64{insn.Bits(73, 1)} << 44) | (u64{insn.Bits(69, 2)} << 45) | (u64{insn.Bits(74, 2)} << 47) | (u64{insn.Bits(71, 1)} << 49)); break; } - case 0x20b: case 0x80b: case 0xa0b: // FSETP + case 0x20b: case 0x80b: case 0xa0b: case 0xc0b: // FSETP case 0x20c: case 0x80c: case 0xa0c: case 0xc0c: { // ISETP const bool floating = (opcode & 0x1ff) == 0xb; - if (!floating && insn.Bits(72, 1)) return fail(index, "native extended integer comparison"); + if (!floating && insn.Bits(72, 1)) { + // ISETP.EX (bit 72): the high-word half of a 64-bit compare. The low words' + // result comes in as the predicate at bits 68-70 (negated by 71), e.g. + // ISETP.NE P0 = lo != 0, then ISETP.NE.EX P0 = (hi != 0) OR P0. EQ: high equal + // AND low; NE: high different OR low; the orderings: high strictly ordered OR + // (high equal AND low). One or two Maxwell ISETPs into the destination; only + // the plain form (no second destination, AND with PT) is handled. + const u32 compare = insn.Bits(76, 3); + const u32 destination = insn.Bits(81, 3); + const u32 low = insn.Bits(68, 4); + if (insn.Bits(84, 3) != 7 || insn.Bits(87, 4) != 7 || insn.Bits(74, 2) != 0 || + compare == 0 || compare == 7 || + (destination != 7 && insn.Bits(12, 3) == destination)) + return fail(index, "native extended integer comparison mode"); + const u32 a = operand(0, false), b = operand(1, false); + const auto isetp = [&](u32 op, u32 input, u32 combine) { + return 0x5b60000000000000ULL | predicate | 7 | (u64{destination} << 3) | + (u64{a} << 8) | (u64{b} << 20) | (u64{input & 7} << 39) | + (u64{input >> 3} << 42) | (u64{combine} << 45) | + (u64{insn.Bits(73, 1)} << 48) | (u64{op} << 49); + }; + constexpr u32 And = 0, Or = 1, Lt = 1, Eq = 2, Gt = 4, Ne = 5; + if (compare == Eq) { + emit(isetp(Eq, low, And)); + } else if (compare == Ne) { + emit(isetp(Ne, low, Or)); + } else { + emit(isetp(Eq, low, And)); + emit(isetp(compare <= 3 ? Lt : Gt, destination, Or)); + } + break; + } const u32 a = operand(0, floating), b = operand(1, floating); const u32 combine = insn.Bits(74, 2); if (combine > 2) return fail(index, "native predicate combination"); @@ -1938,15 +2421,19 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { } case 0x217: case 0x817: case 0xa17: case 0xc17: { // IMNMX register, immediate, cbuf or uniform operand. if ((insn.words[2] & ~(0x04000200U | (opcode == 0xc17 ? 0x08000000U : 0U))) != 0x03800000 || - (opcode != 0xa17 && insn.words[1] > (opcode == 0x817 ? 0x7ffffU : 255U)) || + (opcode == 0x817 && insn.words[1] > 0x7ffffU && insn.words[1] < 0xfff80000U) || + (opcode != 0xa17 && opcode != 0x817 && insn.words[1] > 255U) || (opcode == 0xa17 && (!valid_cbuf() || insn.Bits(62, 2)))) return fail(index, "native integer min/max mode"); + // An immediate is a signed 20-bit value: 19 bits at 20, the sign in bit 56. const u32 source = opcode == 0xc17 ? uniform_register(insn.words[1]) : opcode == 0xa17 ? operand(1, false) + : opcode == 0x817 ? insn.words[1] & 0x7ffffU : insn.words[1]; if (source == ~0U) return fail(index, "native uniform min/max operand"); emit(base(opcode == 0x817 ? 0x3820000000000000ULL : 0x5c20000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{source} << 20) | + (u64{opcode == 0x817 ? insn.words[1] >> 31 : 0U} << 56) | (u64{insn.Bits(87, 4)} << 39) | (u64{insn.Bits(73, 1)} << 48)); break; } @@ -1976,6 +2463,20 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { break; } case 0xa11: { // LEA with constant base and carry predicate. + if ((insn.words[2] & ~(0x07800000U | 0x0000f800U)) == 0x000f06ffU && !insn.Bits(62, 2) && + insn.Bits(75, 5) != 0) { + // LEA.HI.X.SX32, no high source: (sign-extended Ra << shift) >> 32, plus the + // constant high word and the carry predicate (bits 87-90). + const u32 b = operand(1, false); + emit(0x3828000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | + (u64{32 - insn.Bits(75, 5)} << 20) | (u64{1} << 48)); + mov_imm(252, 1); + emit(0x5ca0000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{255} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{b} << 20)); + emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{252} << 20)); + break; + } if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || insn.Bits(81, 3) == 7 || insn.Bits(62, 2)) return fail(index, "native constant LEA carry mode"); @@ -1986,35 +2487,73 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { (dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49)); break; } - case 0x211: case 0x811: { // LEA low word, no carry result + case 0x211: case 0x811: case 0xc11: { // LEA low word (0xc11: uniform Rb) if (insn.Bits(80, 1)) { - // LEA.HI Rd, Ra, Rb (or an immediate, 0x811), RZ, s: Rb + high word of (Ra << s); - // bit 73 sign-extends Ra. - if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(81, 3) != 7 || - insn.Bits(64, 8) != 255 || (opcode == 0x211 && insn.Bits(62, 2))) + // LEA.HI Rd, Ra, Rb (or an immediate, 0x811), Rc, s: Rb + high word of (Rc:Ra << s); + // bit 73 sign-extends Ra (Rc is RZ) and .X (bit 74) adds the carry predicate in bits 87-90. + const bool extended = insn.Bits(74, 1) != 0; + const u32 high = insn.Bits(64, 8); + if (insn.Bits(72, 1) || insn.Bits(81, 3) != 7 || + (high != 255 && insn.Bits(73, 1)) || (opcode == 0x211 && insn.Bits(62, 2)) || + (opcode == 0xc11 && (insn.words[1] & ~0x3fU))) return fail(index, "native high effective address mode"); - u32 b = insn.Bits(32, 8); + u32 b = opcode == 0xc11 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); + if (b == ~0U) return fail(index, "native uniform LEA register"); if (opcode == 0x811) { b = insn.words[1] ? 251U : 255U; if (b == 251) mov_imm(251, insn.words[1]); } const u64 shift = (u64{32 - insn.Bits(75, 5)} << 20) | (u64{insn.Bits(73, 1)} << 48); - if (b == 255) { - emit(base(0x3828000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | shift); + // The high word is a right shift of Ra, or with Rc a left funnel shift (SHF.L). + const auto high_word = [&](u64 target) { + if (high == 255) emit(target | 0x3828000000000000ULL | (u64{insn.Bits(24, 8)} << 8) | shift); + else emit(target | 0x36f8000000000000ULL | (u64{insn.Bits(24, 8)} << 8) | (u64{high} << 39) | + (u64{insn.Bits(75, 5)} << 20)); + }; + if (b == 255 && !extended) { + high_word(base(0)); } else { - emit(0x3828000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | shift); + high_word(predicate | 250); + if (extended) { + mov_imm(252, 1); + emit(0x5ca0000000000000ULL | predicate | 252 | (u64{252} << 8) | (u64{255} << 20) | + (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + emit(0x5c10000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{252} << 20)); + } emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{b} << 20)); } break; } - if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || insn.Bits(81, 3) != 7 || - (opcode == 0x211 && insn.Bits(62, 1))) return fail(index, "native LEA carry/high mode"); + // A carry out (bits 81-83) is the low half of a 64-bit address; IADD3.X adds it next. + const u32 carry_out = insn.Bits(81, 3); + if (insn.Bits(72, 1) || insn.Bits(74, 1) || insn.Bits(80, 1) || + (carry_out != 7 && ((opcode != 0x811 && insn.Bits(63, 1)) || + (rewritten_uniform && (opcode != 0x211 || carry_out > 1)))) || + (opcode == 0x211 && insn.Bits(62, 1)) || (opcode == 0xc11 && (insn.words[1] & ~0x8000003fU))) + return fail(index, "native LEA carry/high mode"); // 0x211 bit 63 negates Rb (LEA Rd, Ra, -Rb, s), as Maxwell ISCADD's bit 48. - u32 b = insn.Bits(32, 8); + u32 b = opcode == 0xc11 ? uniform_register(insn.Bits(32, 8)) : insn.Bits(32, 8); + if (b == ~0U) return fail(index, "native uniform LEA register"); if (opcode == 0x811) { mov_imm(251, insn.words[1]); b = 251; } + if (carry_out != 7) { + // The sum in R250, carry = sum < Rb (unsigned), as IADD3's carry out; a uniform + // predicate goes through P(6-n) into R239. + const u32 target = rewritten_uniform ? 6 - carry_out : carry_out, mask = 1U << target; + emit(0x5c18000000000000ULL | predicate | 250 | (u64{insn.Bits(24, 8)} << 8) | + (u64{b} << 20) | (u64{insn.Bits(75, 5)} << 39)); + if (rewritten_uniform) emit(0x38e8000000070000ULL | 253 | (u64{255} << 8) | (u64{mask} << 20)); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{target} << 3) | (u64{250} << 8) | + (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49)); + if (rewritten_uniform) { + emit(0x38e8000000070000ULL | 239 | (u64{239} << 8) | (u64{mask} << 20)); + emit(0x38f0000000070000ULL | (u64{253} << 8) | (u64{mask} << 20)); + } + emit(base(0x5c98000000000000ULL) | (u64{15} << 39) | (u64{250} << 20)); + break; + } emit(base(0x5c18000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{b} << 20) | (u64{insn.Bits(75, 5)} << 39) | - (u64{opcode == 0x211 ? insn.Bits(63, 1) : 0U} << 48)); + (u64{opcode != 0x811 ? insn.Bits(63, 1) : 0U} << 48)); break; } case 0x210: case 0x810: case 0xa10: case 0xc10: { // IADD3 with zero third source and no carry outputs @@ -2022,7 +2561,7 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { // a carry-in predicate (bits 87-89, negate 90). Uniform forms keep UPn in R239 bit 6-n. if ((opcode == 0x210 || opcode == 0x810) && insn.Bits(64, 8) == 255 && !insn.Bits(73, 1) && !insn.Bits(75, 2) && insn.Bits(84, 3) == 7 && insn.Bits(77, 4) == 15 && !insn.Bits(72, 1) && - (insn.Bits(81, 3) != 7 || insn.Bits(74, 1))) { + (insn.Bits(81, 3) != 7 || insn.Bits(74, 1)) && (opcode == 0x810 || !insn.Bits(63, 1))) { const bool carry_in = insn.Bits(74, 1) != 0; const u32 carry_out = insn.Bits(81, 3); const u32 a = insn.Bits(24, 8); @@ -2065,9 +2604,65 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { emit(base(0x5c10000000000000ULL) | (u64{250} << 8) | (u64{251} << 20)); break; } - if (opcode == 0xa10 && insn.Bits(81, 3) != 7) { + // Three sources, negations or two carries: the sum builds up in R253 (.CC) with the carry + // count in R254 (.X). Pu/Pv (bits 81/84) are set when the count reaches 1/2, and IADD3.X + // adds both carry-ins (bits 87, 77). A negated source is ~x, plus 1 unless extended. + const bool extended = insn.Bits(74, 1) != 0; + const bool negate_a = insn.Bits(72, 1), negate_b = opcode != 0x810 && insn.Bits(63, 1), negate_c = insn.Bits(75, 1); + if ((extended || insn.Bits(81, 6) != 0x3f) && + (insn.Bits(64, 8) != 255 || insn.Bits(84, 3) != 7 || negate_a || negate_b || negate_c || + (extended && insn.Bits(77, 4) != 15))) { + const u32 first_output = insn.Bits(81, 3), second_output = insn.Bits(84, 3); + if (rewritten_uniform || guarded_uniform || insn.Bits(73, 1) || insn.Bits(76, 1) || + (first_output != 7 && first_output == second_output) || + (!extended && (insn.Bits(77, 4) != 15 || insn.Bits(87, 4) != 15)) || + (opcode == 0x210 && (insn.words[1] & ~0x800000ffU)) || (opcode == 0xa10 && insn.Bits(62, 1)) || + (opcode == 0xc10 && (insn.words[1] & ~0x8000003fU))) + return fail(index, "native three-input carry mode"); + u32 a = insn.Bits(24, 8), b = operand(1, false), c = insn.Bits(64, 8); + const auto invert = [&](u32& source, u32 target) { + emit(0x0400000000000000ULL | predicate | target | (u64{source} << 8) | (u64{0xffffffffU} << 20) | (u64{2} << 53)); + source = target; + }; + if (negate_a) invert(a, 250); + if (negate_b) invert(b, 251); + if (negate_c) invert(c, 252); + const auto accumulate = [&](u32 source) { + emit(0x5c10000000000000ULL | predicate | 253 | (u64{253} << 8) | (u64{source} << 20) | (u64{1} << 47)); + emit(0x5c10000000000000ULL | predicate | 254 | (u64{254} << 8) | (u64{255} << 20) | (u64{1} << 43)); + }; + emit(0x5c10000000000000ULL | predicate | 253 | (u64{a} << 8) | (u64{b} << 20) | (u64{1} << 47)); + emit(0x5c10000000000000ULL | predicate | 254 | (u64{255} << 8) | (u64{255} << 20) | (u64{1} << 43)); + accumulate(c); + if (extended) { + for (const u32 carry : {insn.Bits(87, 4), insn.Bits(77, 4)}) { + if (carry == 15) continue; + mov_imm(250, 1); + if (carry != 7) + emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | + (u64{carry & 7} << 39) | (u64{carry >> 3} << 42)); + accumulate(250); + } + } else if (const u32 ones = u32{negate_a} + u32{negate_b} + u32{negate_c}) { + mov_imm(250, ones); + accumulate(250); + } + if (dest != 255) emit(base(0x5c98000000000000ULL) | (u64{253} << 20) | (u64{15} << 39)); + // An output that is also the guard predicate is written last. + const bool guard_first = (native_predicate & 7) != 7 && first_output == (native_predicate & 7); + for (const u32 output : {guard_first ? 1U : 0U, guard_first ? 0U : 1U}) { + const u32 target = output ? second_output : first_output; + if (target == 7) continue; + mov_imm(250, output + 1); + emit(0x5b60000000000000ULL | predicate | 7 | (u64{target} << 3) | (u64{254} << 8) | + (u64{250} << 20) | (u64{7} << 39) | (u64{6} << 49)); + } + break; + } + if ((opcode == 0xa10 || opcode == 0xc10) && insn.Bits(81, 3) != 7) { // cbuf or uniform b if (insn.Bits(64, 8) != 255 || insn.Bits(73, 4) || insn.Bits(84, 7) != 0x7f || - insn.Bits(77, 4) != 15 || insn.Bits(63, 1) || insn.Bits(72, 1)) + insn.Bits(77, 4) != 15 || insn.Bits(63, 1) || insn.Bits(72, 1) || + (opcode == 0xc10 && (insn.words[1] & ~0x3fU))) return fail(index, "native constant add carry mode"); const u32 b = operand(1, false); emit(base(0x5c10000000000000ULL) | (u64{insn.Bits(24, 8)} << 8) | (u64{b} << 20)); @@ -2075,13 +2670,15 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { (dest << 8) | (u64{b} << 20) | (u64{7} << 39) | (u64{1} << 49)); break; } - if (opcode == 0xa10 && insn.Bits(74, 1)) { - if (insn.Bits(24, 8) != 255 || (insn.words[2] & ~0x07800000U) != 0x007fe4ffU) + if ((opcode == 0xa10 || opcode == 0xc10) && insn.Bits(74, 1)) { // IADD3.X Ra + cbuf or UR + carry predicate + if ((insn.words[2] & ~(0x07800000U | (opcode == 0xc10 ? 0x08000000U : 0U))) != 0x007fe4ffU || + (opcode == 0xc10 && (insn.words[1] & ~0x3fU))) return fail(index, "native extended constant add mode"); - const u32 b = operand(1, false); + const u32 a = insn.Bits(24, 8), b = operand(1, false); mov_imm(250, 1); emit(0x5ca0000000000000ULL | predicate | 250 | (u64{250} << 8) | (u64{255} << 20) | (u64{insn.Bits(87, 3)} << 39) | (u64{insn.Bits(90, 1)} << 42)); + if (a != 255) emit(0x5c10000000000000ULL | predicate | 250 | (u64{a} << 8) | (u64{250} << 20)); emit(base(0x5c10000000000000ULL) | (u64{b} << 8) | (u64{250} << 20)); break; } @@ -2214,6 +2811,24 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { return finish(index); // the closing self-branch break; } + case 0x923: case 0x927: { // ISBERD / ISBEWR: task and mesh programs' stage buffer. + // Ampere: Ra 24-31, offset 40-63, ISBERD dest 16-23, ISBEWR data 32-39; 75 = .32 (else + // U8), 76-77 = 3 .ATTR (0 the map region), 78 = .SKEW, 79 = .O (the output buffer). + // Kept as an nx2 pseudo instruction (Maxwell 0xefc0, free there) that the frontend + // reads for the task or mesh stage: data/dest 0-7, Ra 8-15, offset 20-43, .32 44, + // .ATTR 45, .SKEW 46, .O 47, write 48. Addresses are linear bytes in the region. + const bool write = opcode == 0x927; + const u32 region = insn.Bits(76, 2); + const bool output = insn.Bits(79, 1) != 0; + if ((insn.words[2] & ~0xf800U) != 0 || (region != 0 && region != 3) || write != output || + (!write && region != 3) || (!write && dest == 255)) + return fail(index, "native stage buffer access mode"); + emit(0xefc0000000000000ULL | predicate | (write ? u64{insn.Bits(32, 8)} : dest) | + (u64{insn.Bits(24, 8)} << 8) | (u64{insn.Bits(40, 24)} << 20) | + (u64{insn.Bits(75, 1)} << 44) | (u64{region == 3} << 45) | + (u64{insn.Bits(78, 1)} << 46) | (u64{output} << 47) | (u64{write} << 48)); + break; + } case 0x94d: { // EXIT if (insn.Bits(84, 2)) { return fail(index, "native conditional/special EXIT"); @@ -2264,4 +2879,14 @@ inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { return fail(static_cast<u32>(code.size()), main_exited ? "native program lacks captured library return" : "native program has no unconditional EXIT"); } +inline LoweredProgram LowerToMaxwell(std::span<const Instruction> code) { + // Programs using uniform registers above UR27 are lowered again with mirrors for them. + u32 high_uniforms = 0; + for (;;) { + LoweredProgram lowered = LowerToMaxwellWith(code, high_uniforms); + if (lowered || lowered.high_uniform_registers <= high_uniforms) return lowered; + high_uniforms = lowered.high_uniform_registers; + } +} + } // namespace Shader::Ampere diff --git a/src/shader_recompiler/frontend/ir/ir_emitter.cpp b/src/shader_recompiler/frontend/ir/ir_emitter.cpp index 95815d7d51..23d1db5bf4 100644 --- a/src/shader_recompiler/frontend/ir/ir_emitter.cpp +++ b/src/shader_recompiler/frontend/ir/ir_emitter.cpp @@ -329,6 +329,22 @@ void IREmitter::SetPatch(Patch patch, const F32& value) { Inst(Opcode::SetPatch, patch, value); } +U32 IREmitter::IsbeRead(const U32& address) { + return Inst<U32>(Opcode::IsbeRead, address); +} + +void IREmitter::IsbeWriteAttribute(const U32& address, const U32& value) { + Inst(Opcode::IsbeWriteAttribute, address, value); +} + +void IREmitter::IsbeWriteMapU8(const U32& address, const U32& value) { + Inst(Opcode::IsbeWriteMapU8, address, value); +} + +void IREmitter::IsbeWriteMapU32(const U32& address, const U32& value) { + Inst(Opcode::IsbeWriteMapU32, address, value); +} + void IREmitter::SetFragColor(u32 index, u32 component, const F32& value) { Inst(Opcode::SetFragColor, Imm32(index), Imm32(component), value); } diff --git a/src/shader_recompiler/frontend/ir/ir_emitter.h b/src/shader_recompiler/frontend/ir/ir_emitter.h index f22f0f75e2..d0cd88ca3c 100644 --- a/src/shader_recompiler/frontend/ir/ir_emitter.h +++ b/src/shader_recompiler/frontend/ir/ir_emitter.h @@ -88,6 +88,11 @@ public: [[nodiscard]] F32 GetPatch(Patch patch); void SetPatch(Patch patch, const F32& value); + [[nodiscard]] U32 IsbeRead(const U32& address); + void IsbeWriteAttribute(const U32& address, const U32& value); + void IsbeWriteMapU8(const U32& address, const U32& value); + void IsbeWriteMapU32(const U32& address, const U32& value); + void SetFragColor(u32 index, u32 component, const F32& value); void SetSampleMask(const U32& value); void SetFragDepth(const F32& value); diff --git a/src/shader_recompiler/frontend/ir/microinstruction.cpp b/src/shader_recompiler/frontend/ir/microinstruction.cpp index 84417980bc..a579ed1057 100644 --- a/src/shader_recompiler/frontend/ir/microinstruction.cpp +++ b/src/shader_recompiler/frontend/ir/microinstruction.cpp @@ -82,6 +82,9 @@ bool Inst::MayHaveSideEffects() const noexcept { case Opcode::SetAttribute: case Opcode::SetAttributeIndexed: case Opcode::SetPatch: + case Opcode::IsbeWriteAttribute: + case Opcode::IsbeWriteMapU8: + case Opcode::IsbeWriteMapU32: case Opcode::SetFragColor: case Opcode::SetSampleMask: case Opcode::SetFragDepth: diff --git a/src/shader_recompiler/frontend/ir/opcodes.cpp b/src/shader_recompiler/frontend/ir/opcodes.cpp index 94ba0f6ae0..1999f877fe 100644 --- a/src/shader_recompiler/frontend/ir/opcodes.cpp +++ b/src/shader_recompiler/frontend/ir/opcodes.cpp @@ -10,7 +10,7 @@ namespace Shader::IR { namespace Detail { -OpcodeMeta META_TABLE[534] = { +OpcodeMeta META_TABLE[538] = { #define OPCODE(name_token, type_token, ...) \ { \ .name{#name_token}, \ @@ -21,7 +21,7 @@ OpcodeMeta META_TABLE[534] = { #undef OPCODE }; -u8 NUM_ARGS[534] = { +u8 NUM_ARGS[538] = { #define OPCODE(name_token, type_token, ...) u8(CalculateNumArgsOf(Opcode::name_token)), #include "opcodes.inc" #undef OPCODE diff --git a/src/shader_recompiler/frontend/ir/opcodes.h b/src/shader_recompiler/frontend/ir/opcodes.h index 50f9d7b155..c59f50adfe 100644 --- a/src/shader_recompiler/frontend/ir/opcodes.h +++ b/src/shader_recompiler/frontend/ir/opcodes.h @@ -57,12 +57,12 @@ static constexpr Type F64x2{Type::F64x2}; static constexpr Type F64x3{Type::F64x3}; static constexpr Type F64x4{Type::F64x4}; -extern OpcodeMeta META_TABLE[534]; +extern OpcodeMeta META_TABLE[538]; constexpr size_t CalculateNumArgsOf(Opcode op) noexcept { const auto& arg_types = META_TABLE[size_t(op)].arg_types; return size_t(std::distance(arg_types.begin(), std::ranges::find(arg_types, Type::Void))); } -extern u8 NUM_ARGS[534]; +extern u8 NUM_ARGS[538]; } // namespace Detail /// Get return type of an opcode diff --git a/src/shader_recompiler/frontend/ir/opcodes.inc b/src/shader_recompiler/frontend/ir/opcodes.inc index fee0afaf33..552ec4fbb6 100644 --- a/src/shader_recompiler/frontend/ir/opcodes.inc +++ b/src/shader_recompiler/frontend/ir/opcodes.inc @@ -48,6 +48,10 @@ OPCODE(GetAttributeIndexed, F32, U32, OPCODE(SetAttributeIndexed, Void, U32, F32, U32, ) OPCODE(GetPatch, F32, Patch, ) OPCODE(SetPatch, Void, Patch, F32, ) +OPCODE(IsbeRead, U32, U32, ) +OPCODE(IsbeWriteAttribute, Void, U32, U32, ) +OPCODE(IsbeWriteMapU8, Void, U32, U32, ) +OPCODE(IsbeWriteMapU32, Void, U32, U32, ) OPCODE(SetFragColor, Void, U32, U32, F32, ) OPCODE(SetSampleMask, Void, U32, ) OPCODE(SetFragDepth, Void, F32, ) diff --git a/src/shader_recompiler/frontend/ir/program.h b/src/shader_recompiler/frontend/ir/program.h index 6b4a05c598..fee5cee721 100644 --- a/src/shader_recompiler/frontend/ir/program.h +++ b/src/shader_recompiler/frontend/ir/program.h @@ -27,6 +27,7 @@ struct Program { u32 local_memory_size{}; u32 shared_memory_size{}; bool is_geometry_passthrough{}; + MeshInfo mesh{}; }; [[nodiscard]] std::string DumpProgram(const Program& program); diff --git a/src/shader_recompiler/frontend/maxwell/maxwell.inc b/src/shader_recompiler/frontend/maxwell/maxwell.inc index e5dc9c8005..5facfb4803 100644 --- a/src/shader_recompiler/frontend/maxwell/maxwell.inc +++ b/src/shader_recompiler/frontend/maxwell/maxwell.inc @@ -55,6 +55,7 @@ INST(IMNMX_reg, "IMNMX (reg)", "0101 1100 0010 0---") INST(IMUL_cbuf, "IMUL (cbuf)", "0100 1100 0011 1---") INST(IMUL_reg, "IMUL (reg)", "0101 1100 0011 1---") INST(ISBERD, "ISBERD", "1110 1111 1101 0---") +INST(ISBE_nx2, "ISBE (nx2)", "1110 1111 1100 0---") INST(ISCADD_cbuf, "ISCADD (cbuf)", "0100 1100 0001 1---") INST(ISCADD_reg, "ISCADD (reg)", "0101 1100 0001 1---") INST(LDC, "LDC", "1110 1111 1001 0---") diff --git a/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp b/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp index 329d44443e..50cff52f88 100644 --- a/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp +++ b/src/shader_recompiler/frontend/maxwell/structured_control_flow.cpp @@ -314,7 +314,8 @@ bool NeedsLift(Node goto_stmt, Node label_stmt) noexcept { class GotoPass { public: - explicit GotoPass(Flow::CFG& cfg, ObjectPool<Statement>& stmt_pool) : pool{stmt_pool} { + explicit GotoPass(Flow::CFG& cfg, ObjectPool<Statement>& stmt_pool, bool converge_exits_) + : pool{stmt_pool}, converge_exits{converge_exits_} { std::vector gotos{BuildTree(cfg)}; const auto end{gotos.rend()}; for (auto goto_stmt = gotos.rbegin(); goto_stmt != end; ++goto_stmt) { @@ -381,7 +382,17 @@ private: u32 label_id{0}; std::vector<Node> gotos; Flow::Function& first_function{cfg.Functions().front()}; - BuildTree(cfg, first_function, label_id, gotos, root_stmt.children.end(), std::nullopt); + Node insert_point{root_stmt.children.end()}; + if (converge_exits) { + // Every EXIT jumps to one return at the end, so all invocations reach the same + // epilogue (task and mesh programs emit their outputs there, after a barrier). + Tree& root{root_stmt.children}; + exit_label = root.insert(root.end(), *pool.Create(Label{}, label_id, &root_stmt)); + ++label_id; + root.insert(root.end(), *pool.Create(Return{}, &root_stmt)); + insert_point = *exit_label; + } + BuildTree(cfg, first_function, label_id, gotos, insert_point, std::nullopt); return gotos; } @@ -454,7 +465,14 @@ private: break; } case Flow::EndClass::Exit: - root.insert(ip, *pool.Create(Return{}, &root_stmt)); + if (exit_label) { + Statement* const always_cond{ + pool.Create(Identity{}, IR::Condition{true}, &root_stmt)}; + gotos.push_back(root.insert( + ip, *pool.Create(Goto{}, always_cond, *exit_label, &root_stmt))); + } else { + root.insert(ip, *pool.Create(Return{}, &root_stmt)); + } break; case Flow::EndClass::Return: { Statement* const always_cond{pool.Create(Identity{}, block.cond, &root_stmt)}; @@ -618,6 +636,8 @@ private: ObjectPool<Statement>& pool; Statement root_stmt{FunctionTag{}}; + bool converge_exits{}; + std::optional<Node> exit_label; }; [[nodiscard]] Statement* TryFindForwardBlock(Statement& stmt) { @@ -987,7 +1007,8 @@ IR::AbstractSyntaxList BuildASL(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR:: Environment& env, Flow::CFG& cfg, const HostTranslateInfo& host_info) { ObjectPool<Statement> stmt_pool{64}; - GotoPass goto_pass{cfg, stmt_pool}; + const Stage stage{env.ShaderStage()}; + GotoPass goto_pass{cfg, stmt_pool, stage == Stage::Task || stage == Stage::Mesh}; Statement& root{goto_pass.RootStatement()}; IR::AbstractSyntaxList syntax_list; TranslatePass{inst_pool, block_pool, stmt_pool, env, root, syntax_list, host_info}; diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp b/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp index 46660563a1..27de2a0171 100644 --- a/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp @@ -125,4 +125,40 @@ void TranslatorVisitor::ISBERD(u64 insn) { X(isberd.dest_reg.Value(), X(isberd.src_reg.Value())); } +// nx2 pseudo instruction for the ISBERD/ISBEWR of Switch 2 task and mesh programs (see +// frontend/ampere/lower_to_maxwell.h). Addresses are linear bytes in the stage buffer: reads come +// from the input attribute region, writes go to the output attribute or map region. +void TranslatorVisitor::ISBE_nx2(u64 insn) { + union { + u64 raw; + BitField<0, 8, IR::Reg> reg; + BitField<8, 8, IR::Reg> address_reg; + BitField<20, 24, u64> offset; + BitField<44, 1, u64> is_32; + BitField<45, 1, u64> attribute; + BitField<48, 1, u64> write; + } const isbe{insn}; + + const Stage stage{env.ShaderStage()}; + if (stage != Stage::Task && stage != Stage::Mesh) { + throw NotImplementedException("Stage buffer access outside task and mesh programs"); + } + const IR::U32 address{ir.IAdd(X(isbe.address_reg), ir.Imm32(static_cast<u32>(isbe.offset)))}; + if (isbe.write == 0) { + X(isbe.reg, ir.IsbeRead(address)); + return; + } + const IR::U32 value{X(isbe.reg)}; + if (isbe.attribute != 0) { + if (isbe.is_32 == 0) { + throw NotImplementedException("U8 stage buffer attribute write"); + } + ir.IsbeWriteAttribute(address, value); + } else if (isbe.is_32 != 0) { + ir.IsbeWriteMapU32(address, value); + } else { + ir.IsbeWriteMapU8(address, value); + } +} + } // namespace Shader::Maxwell diff --git a/src/shader_recompiler/frontend/maxwell/translate_program.cpp b/src/shader_recompiler/frontend/maxwell/translate_program.cpp index cd4678cef6..fe6d8d8913 100644 --- a/src/shader_recompiler/frontend/maxwell/translate_program.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate_program.cpp @@ -101,10 +101,12 @@ void AddNVNStorageBuffers(IR::Program& program) { switch (program.stage) { case Stage::VertexA: case Stage::VertexB: + case Stage::Task: // bound in the vertex slot return 0x110u; case Stage::TessellationControl: return 0x210u; case Stage::TessellationEval: + case Stage::Mesh: // bound in the tessellation slot return 0x310u; case Stage::Geometry: return 0x410u; @@ -304,6 +306,30 @@ void LowerGeometryPassthrough(const IR::Program& program, const HostTranslateInf } } +// The lines of a 32-element group in a mesh program's stage buffer: one per attribute an output +// map enables, in attribute order (system values, then generics). +void AddMeshLines(const ProgramHeader& sph, boost::container::static_vector<u32, 136>& lines) { + const auto& c{sph.vtg.omap_systemc}; + if (c.clip_distances != 0 || c.point_sprite_s != 0 || c.point_sprite_t != 0 || + c.fog_coordinate != 0 || c.tessellation_eval_point_u != 0 || + c.tessellation_eval_point_v != 0 || c.instance_id != 0 || c.vertex_id != 0) { + throw NotImplementedException("Mesh outputs past the generic attributes"); + } + for (u32 bit = 0; bit < 8; ++bit) { + if (((sph.vtg.omap_systemb.raw >> bit) & 1) != 0) { + lines.push_back(static_cast<u32>(IR::Attribute::PrimitiveId) + bit); + } + } + for (u32 generic = 0; generic < 32; ++generic) { + const std::array<bool, 4> mask{sph.vtg.OutputGeneric(generic)}; + for (u32 element = 0; element < 4; ++element) { + if (mask[element]) { + lines.push_back(static_cast<u32>(IR::Attribute::Generic0X) + generic * 4 + element); + } + } + } +} + void TightenOutputVertices(IR::Program& program) { if (program.stage != Stage::Geometry || program.is_geometry_passthrough) { return; @@ -368,6 +394,30 @@ IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Blo program.workgroup_size = env.WorkgroupSize(); program.shared_memory_size = env.SharedMemorySize(); break; + case Stage::Task: + case Stage::Mesh: { + // SET_MESH_SHADER_A: topology 3:0, max vertex 12:4, max primitive 22:13; + // SET_MESH_SHADER_B: mesh threads 25:20; SET_MESH_INIT_SHADER: task threads 5:0, local + // buffer lines 20:12, lines passed to the mesh stage 30:22 (lines are 128 bytes). + const auto& registers{env.MeshRegisters()}; + const u32 shader_a{registers[0]}, shader_b{registers[1]}, init{registers[2]}; + const u32 lines{(std::max)((init >> 12) & 0x1ff, (init >> 22) & 0x1ff)}; + program.mesh.payload_words = (std::max)(lines, 1U) * 32; + program.mesh.topology = shader_a & 0xf; + program.mesh.max_vertices = (shader_a >> 4) & 0x1ff; + program.mesh.max_primitives = (shader_a >> 13) & 0x3ff; + const u32 threads{program.stage == Stage::Task ? init & 0x3f : (shader_b >> 20) & 0x3f}; + program.workgroup_size = {(std::max)(threads, 1U), 1, 1}; + if (program.stage == Stage::Mesh) { + if (program.mesh.topology != 4) { + throw NotImplementedException("Mesh output topology {} (SET_MESH_SHADER_A {:#010x})", + program.mesh.topology, shader_a); + } + AddMeshLines(env.SPH(), program.mesh.vertex_lines); + AddMeshLines(env.MeshPrimitiveHeader(), program.mesh.primitive_lines); + } + break; + } default: break; } @@ -406,6 +456,18 @@ IR::Program TranslateProgram(ObjectPool<IR::Inst>& inst_pool, ObjectPool<IR::Blo Optimization::VerificationPass(program); } Optimization::CollectShaderInfoPass(env, program); + program.info.is_task = program.stage == Stage::Task; + program.info.is_mesh = program.stage == Stage::Mesh; + if (program.stage == Stage::Mesh) { + // Its outputs come from the stage buffer lines, not attribute stores; the next stage + // reads what they name. + for (const u32 line : program.mesh.vertex_lines) { + program.info.stores.Set(static_cast<IR::Attribute>(line), true); + } + for (const u32 line : program.mesh.primitive_lines) { + program.info.stores.Set(static_cast<IR::Attribute>(line), true); + } + } Optimization::LayerPass(program, normalized_host_info); Optimization::VendorWorkaroundPass(program); diff --git a/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp b/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp index 9010c2bf68..c212a25d37 100644 --- a/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp +++ b/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp @@ -963,10 +963,12 @@ void CollectShaderInfoPass(Environment& env, IR::Program& program) { switch (program.stage) { case Stage::VertexA: case Stage::VertexB: + case Stage::Task: // bound in the vertex slot return 0x110u; case Stage::TessellationControl: return 0x210u; case Stage::TessellationEval: + case Stage::Mesh: // bound in the tessellation slot return 0x310u; case Stage::Geometry: return 0x410u; @@ -978,6 +980,11 @@ void CollectShaderInfoPass(Environment& env, IR::Program& program) { throw InvalidArgument("Invalid stage {}", program.stage); }()}; info.nvn_buffer_base = base; + if (program.stage == Stage::Task || program.stage == Stage::Mesh) { + // The stage buffer reads the workgroup index; the epilogue splits work by invocation. + info.uses_workgroup_id = true; + info.uses_local_invocation_id = true; + } for (IR::Block* const block : program.post_order_blocks) { for (IR::Inst& inst : block->Instructions()) { diff --git a/src/shader_recompiler/ir_opt/position_pass.cpp b/src/shader_recompiler/ir_opt/position_pass.cpp index 4890346944..600a596bf3 100644 --- a/src/shader_recompiler/ir_opt/position_pass.cpp +++ b/src/shader_recompiler/ir_opt/position_pass.cpp @@ -83,6 +83,37 @@ void BlitConstantsPass(Environment& env, IR::Program& program) { if (env.ShaderStage() != Stage::VertexB && env.ShaderStage() != Stage::Fragment) { return; } + // Bank 24 is root table 0, which games may load themselves (3D methods 0x504/0x508; the buffer + // cache binds it as bank 17): Fire Emblem: Fortune's Weave keeps its material push constants + // there. A shader reading bank 17 outside the blit block reads such a table and keeps its reads, + // as does one reading only words the blit block leaves zero (an unbound bank reads zero anyway; + // the game's enemy health gauges read only 0x40 of their table). + const auto in_blit_block = [](u32 offset) { + return offset < 0x20 || (offset >= 0x30 && offset < 0x4c) || (offset >= 0x50 && offset < 0x60); + }; + const auto is_nonzero_word = [](u32 offset) { + return offset < 0x20 || offset == 0x38 || offset == 0x3c || offset == 0x58 || offset == 0x5c; + }; + bool reads_nonzero_word{}; + for (IR::Block* const block : program.post_order_blocks) { + for (IR::Inst& inst : block->Instructions()) { + const IR::Opcode opcode{inst.GetOpcode()}; + if ((opcode != IR::Opcode::GetCbufU32 && opcode != IR::Opcode::GetCbufF32 && + opcode != IR::Opcode::GetCbufU32x2) || + !inst.Arg(0).IsImmediate() || inst.Arg(0).U32() != 17 || !inst.Arg(1).IsImmediate()) { + continue; + } + const u32 offset{inst.Arg(1).U32()}; + const bool pair{opcode == IR::Opcode::GetCbufU32x2}; + if (!in_blit_block(offset) || (pair && !in_blit_block(offset + 4))) { + return; + } + reads_nonzero_word |= is_nonzero_word(offset) || (pair && is_nonzero_word(offset + 4)); + } + } + if (!reads_nonzero_word) { + return; + } bool used{}; for (IR::Block* const block : program.post_order_blocks) { for (IR::Inst& inst : block->Instructions()) { diff --git a/src/shader_recompiler/ir_opt/texture_pass.cpp b/src/shader_recompiler/ir_opt/texture_pass.cpp index fae69ab91b..8a72f565dc 100644 --- a/src/shader_recompiler/ir_opt/texture_pass.cpp +++ b/src/shader_recompiler/ir_opt/texture_pass.cpp @@ -5,6 +5,10 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <algorithm> +#include <queue> +#include <unordered_map> +#include <unordered_set> +#include <vector> #include <bit> #include <optional> #include "common/container/unordered_map.h" @@ -340,15 +344,20 @@ static inline u32 GetTextureHandleCached(Environment& env, const ConstBufferAddr } // Cached variants of existing helpers +// A handle computed per draw has no descriptor to read at translation time: assume an ordinary +// floating-point 2D texture, the instruction's own type deciding the rest. static inline TextureType ReadTextureTypeCached(Environment& env, const ConstBufferAddr& cbuf) { + if (cbuf.handle_program != ~0U) return TextureType::Color2D; return env.ReadTextureType(GetTextureHandleCached(env, cbuf)); } static inline TexturePixelFormat ReadTexturePixelFormatCached(Environment& env, const ConstBufferAddr& cbuf) { + if (cbuf.handle_program != ~0U) return TexturePixelFormat::A8B8G8R8_UNORM; return env.ReadTexturePixelFormat(GetTextureHandleCached(env, cbuf)); } static inline bool IsTexturePixelFormatIntegerCached(Environment& env, const ConstBufferAddr& cbuf) { + if (cbuf.handle_program != ~0U) return false; return env.IsTexturePixelFormatInteger(GetTextureHandleCached(env, cbuf)); } @@ -366,12 +375,389 @@ static inline std::optional<ConstBufferAddr> TrackCached(const IR::Value& v, Env std::optional<ConstBufferAddr> TryGetConstBuffer(const IR::Inst* inst, Environment& env, const HostTranslateInfo& host_info, const IR::Block* block); +bool IsMemoryLoad(IR::Opcode opcode) { + switch (opcode) { + case IR::Opcode::LoadGlobalU8: case IR::Opcode::LoadGlobalS8: case IR::Opcode::LoadGlobalU16: + case IR::Opcode::LoadGlobalS16: case IR::Opcode::LoadGlobal32: case IR::Opcode::LoadGlobal64: + case IR::Opcode::LoadGlobal128: case IR::Opcode::LoadStorageU8: case IR::Opcode::LoadStorageS8: + case IR::Opcode::LoadStorageU16: case IR::Opcode::LoadStorageS16: case IR::Opcode::LoadStorage32: + case IR::Opcode::LoadStorage64: case IR::Opcode::LoadStorage128: case IR::Opcode::LoadLocal: + case IR::Opcode::LoadSharedU8: case IR::Opcode::LoadSharedS8: case IR::Opcode::LoadSharedU16: + case IR::Opcode::LoadSharedS16: case IR::Opcode::LoadSharedU32: case IR::Opcode::LoadSharedU64: + case IR::Opcode::LoadSharedU128: + return true; + default: + return false; + } +} + std::optional<ConstBufferAddr> Track(const IR::Value& value, Environment& env, const HostTranslateInfo& host_info, const IR::Block* block) { - return IR::BreadthFirstSearch(value, [&env, &host_info, block](const IR::Inst* inst) { - return TryGetConstBuffer(inst, env, host_info, block); - }, [&](IR::Value operand) { return ResolveConditionalPhi(operand, block); }); + // A value loaded from memory is not a constant buffer word, whatever its address is built + // from: the search does not look into a load's operands (a handle read from a storage + // buffer would otherwise be taken from the buffer's address in the constant buffer). + const IR::Value start{ResolveConditionalPhi(value, block)}; + if (start.IsImmediate()) { + return std::nullopt; + } + std::unordered_set<const IR::Inst*> visited; + std::queue<const IR::Inst*> queue; + queue.push(start.InstRecursive()); + visited.insert(start.InstRecursive()); + while (!queue.empty()) { + const IR::Inst* const inst{queue.front()}; + queue.pop(); + if (const std::optional result = TryGetConstBuffer(inst, env, host_info, block)) { + return result; + } + if (IsMemoryLoad(inst->GetOpcode())) { + continue; + } + for (size_t arg = inst->NumArgs(); arg--;) { + const IR::Value arg_value{ResolveConditionalPhi(inst->Arg(arg), block)}; + if (arg_value.IsImmediate()) { + continue; + } + const IR::Inst* const arg_inst{arg_value.InstRecursive()}; + if (visited.insert(arg_inst).second) { + queue.push(arg_inst); + } + } + } + return std::nullopt; +} + +// Builds a handle program (shader_info.h) for a value computed from constant buffer and storage +// buffer words; false when the value depends on anything else. +thread_local IR::Opcode g_handle_failure{IR::Opcode::Void}; // the first operation not supported + +bool IsUndefined(IR::Opcode opcode) { + return opcode == IR::Opcode::UndefU1 || opcode == IR::Opcode::UndefU8 || + opcode == IR::Opcode::UndefU16 || opcode == IR::Opcode::UndefU32 || + opcode == IR::Opcode::UndefU64; +} + +// Values already emitted by the program being built, by the slot that keeps them, and how deep +// in nested BuildHandleProgram calls the build is. +thread_local std::unordered_map<const IR::Inst*, u32> g_handle_slots; +thread_local int g_handle_depth{}; + +bool BuildHandleNode(const IR::Value& value, std::vector<u32>& code, int& budget, + const IR::Block* block); + +bool BuildHandleProgram(const IR::Value& value, std::vector<u32>& code, int& budget, + const IR::Block* block) { + using Op = HandleOp; + if (value.IsImmediate()) { + return BuildHandleNode(value, code, budget, block); + } + const bool outermost = g_handle_depth++ == 0; + if (outermost) { + g_handle_slots.clear(); + } + const IR::Inst* const inst{value.InstRecursive()}; + bool result; + if (const auto it = g_handle_slots.find(inst); it != g_handle_slots.end()) { + if (it->second == ~0U) { + // Reached again while building it: a loop carries the value, which no single + // evaluation can give. + g_handle_failure = IR::Opcode::Phi; + result = false; + } else { + code.push_back(static_cast<u32>(Op::Load)); + code.push_back(it->second); + result = true; + } + } else { + g_handle_slots.emplace(inst, ~0U); + const size_t start = code.size(); + result = BuildHandleNode(value, code, budget, block); + g_handle_slots.erase(inst); + if (!result) { + code.resize(start); + } + if (result && inst->UseCount() > 1 && g_handle_slots.size() < 256) { + const u32 slot = static_cast<u32>( + std::ranges::count_if(g_handle_slots, [](const auto& entry) { return entry.second != ~0U; })); + g_handle_slots.emplace(inst, slot); + code.push_back(static_cast<u32>(Op::Tee)); + code.push_back(slot); + } + } + --g_handle_depth; + if (outermost) { + g_handle_slots.clear(); + } + return result; } +bool BuildHandleNode(const IR::Value& value, std::vector<u32>& code, int& budget, + const IR::Block* block) { + if (--budget < 0) { + return false; + } + using Op = HandleOp; + const auto emit = [&](Op op) { code.push_back(static_cast<u32>(op)); }; + if (value.IsImmediate()) { + emit(Op::Immediate); + code.push_back(value.Type() == IR::Type::U1 ? (value.U1() ? 1U : 0U) : value.U32()); + return true; + } + const IR::Inst* const inst{value.InstRecursive()}; + const auto build = [&](size_t arg) { return BuildHandleProgram(inst->Arg(arg), code, budget, block); }; + const auto binary = [&](Op op) { + if (!build(0) || !build(1)) return false; + emit(op); + return true; + }; + const auto unary = [&](Op op) { + if (!build(0)) return false; + emit(op); + return true; + }; + // A word of a constant buffer read or of a storage buffer load, at byte offset +extra. + const auto load_word = [&](const IR::Inst* source, u32 extra) { + const auto add_extra = [&] { + if (extra != 0) { + emit(Op::Immediate); + code.push_back(extra); + emit(Op::Add); + } + }; + switch (source->GetOpcode()) { + case IR::Opcode::GetCbufU32: + case IR::Opcode::GetCbufU32x2: { + const IR::Value index{source->Arg(0)}; + const IR::Value offset{source->Arg(1)}; + if (!index.IsImmediate()) return false; + if (offset.IsImmediate()) { + emit(Op::Cbuf); + code.push_back(index.U32()); + code.push_back(offset.U32() + extra); + return true; + } + if (!BuildHandleProgram(offset, code, budget, block)) return false; + add_extra(); + emit(Op::CbufDynamic); + code.push_back(index.U32()); + return true; + } + case IR::Opcode::LoadStorage32: + case IR::Opcode::LoadStorage64: + case IR::Opcode::LoadStorage128: { + const IR::Value binding{source->Arg(0)}; + if (!binding.IsImmediate()) return false; + if (!BuildHandleProgram(source->Arg(1), code, budget, block)) return false; + add_extra(); + emit(Op::Storage); + code.push_back(binding.U32()); + return true; + } + default: + return false; + } + }; + switch (inst->GetOpcode()) { + case IR::Opcode::Identity: + case IR::Opcode::ConditionRef: + case IR::Opcode::ShuffleIndex: // a value made uniform (R2UR): the same in every lane + case IR::Opcode::VoteAny: // of a condition on per-draw words only, the same in every lane + case IR::Opcode::VoteAll: + return build(0); + case IR::Opcode::GetCbufU32: + case IR::Opcode::LoadStorage32: + return load_word(inst, 0); + case IR::Opcode::CompositeExtractU32x2: { + const IR::Value element{inst->Arg(1)}; + if (!element.IsImmediate() || inst->Arg(0).IsImmediate()) return false; + return load_word(inst->Arg(0).InstRecursive(), element.U32() * 4); + } + case IR::Opcode::Phi: { + // Keep the incoming value that is defined; of two, the one made uniform (the + // register-to-uniform move a conditional block makes for the handle). + const IR::Value resolved{ResolveConditionalPhi(value, block)}; + if (resolved.IsImmediate() || resolved.InstRecursive() != inst) { + return BuildHandleProgram(resolved, code, budget, block); + } + std::optional<size_t> chosen; + for (size_t arg = 0; arg < inst->NumArgs(); ++arg) { + const IR::Value incoming{inst->Arg(arg)}; + if (!incoming.IsImmediate() && IsUndefined(incoming.InstRecursive()->GetOpcode())) { + continue; + } + IR::Value probe{incoming}; + bool shuffled = false; + for (int depth = 0; depth < 8 && !probe.IsImmediate(); ++depth) { + const IR::Inst* const probe_inst{probe.InstRecursive()}; + if (probe_inst->GetOpcode() == IR::Opcode::ShuffleIndex) { + shuffled = true; + break; + } + if (probe_inst->GetOpcode() != IR::Opcode::Identity) break; + probe = probe_inst->Arg(0); + } + if (!chosen || shuffled) chosen = arg; + } + if (chosen && build(*chosen)) { + return true; + } + for (size_t arg = 0; arg < inst->NumArgs(); ++arg) { + const IR::Value incoming{inst->Arg(arg)}; + if (arg == chosen || (!incoming.IsImmediate() && IsUndefined(incoming.InstRecursive()->GetOpcode()))) { + continue; + } + if (build(arg)) { + return true; + } + } + return false; + } + case IR::Opcode::IAdd32: return binary(Op::Add); + case IR::Opcode::ISub32: return binary(Op::Sub); + case IR::Opcode::IMul32: return binary(Op::Mul); + case IR::Opcode::BitwiseAnd32: return binary(Op::And); + case IR::Opcode::BitwiseOr32: return binary(Op::Or); + case IR::Opcode::BitwiseXor32: return binary(Op::Xor); + case IR::Opcode::ShiftLeftLogical32: return binary(Op::Shl); + case IR::Opcode::ShiftRightLogical32: return binary(Op::Shr); + case IR::Opcode::ShiftRightArithmetic32: return binary(Op::Sar); + case IR::Opcode::INeg32: return unary(Op::Neg); + case IR::Opcode::BitwiseNot32: return unary(Op::Not); + case IR::Opcode::UMin32: return binary(Op::UMin); + case IR::Opcode::UMax32: return binary(Op::UMax); + case IR::Opcode::SMin32: return binary(Op::SMin); + case IR::Opcode::SMax32: return binary(Op::SMax); + case IR::Opcode::SLessThan: return binary(Op::SLt); + case IR::Opcode::SLessThanEqual: return binary(Op::SLe); + case IR::Opcode::SGreaterThan: return binary(Op::SGt); + case IR::Opcode::SGreaterThanEqual: return binary(Op::SGe); + case IR::Opcode::ULessThan: return binary(Op::ULt); + case IR::Opcode::ULessThanEqual: return binary(Op::ULe); + case IR::Opcode::UGreaterThan: return binary(Op::UGt); + case IR::Opcode::UGreaterThanEqual: return binary(Op::UGe); + case IR::Opcode::IEqual: return binary(Op::Eq); + case IR::Opcode::INotEqual: return binary(Op::Ne); + case IR::Opcode::LogicalAnd: return binary(Op::LAnd); + case IR::Opcode::LogicalOr: return binary(Op::LOr); + case IR::Opcode::LogicalXor: return binary(Op::LXor); + case IR::Opcode::LogicalNot: return unary(Op::LNot); + case IR::Opcode::GetZeroFromOp: // the flag a producing operation sets on its result + if (!build(0)) return false; + emit(Op::Immediate); + code.push_back(0); + emit(Op::Eq); + return true; + case IR::Opcode::GetSignFromOp: + if (!build(0)) return false; + emit(Op::Immediate); + code.push_back(0); + emit(Op::SLt); + return true; + case IR::Opcode::GetCarryFromOp: { + // The carry of a 32-bit addition: the sum below an operand. + const IR::Value op{inst->Arg(0)}; + if (op.IsImmediate() || op.InstRecursive()->GetOpcode() != IR::Opcode::IAdd32) { + g_handle_failure = inst->GetOpcode(); + return false; + } + const IR::Inst* const add{op.InstRecursive()}; + if (!BuildHandleProgram(op, code, budget, block) || + !BuildHandleProgram(add->Arg(0), code, budget, block)) { + return false; + } + emit(Op::ULt); + return true; + } + case IR::Opcode::BitCastU32F32: + case IR::Opcode::BitCastF32U32: + return build(0); + case IR::Opcode::UndefU1: + case IR::Opcode::UndefU32: + // Read before any write on some path: no value to compute, take zero. + emit(Op::Immediate); + code.push_back(0); + return true; + case IR::Opcode::BitFieldUExtract: { + // (base >> offset) & ((1 << count) - 1), the shift by 32 giving the all-ones mask. + const auto mask = [&] { + emit(Op::Immediate); + code.push_back(1); + if (!build(2)) return false; + emit(Op::Shl); + emit(Op::Immediate); + code.push_back(1); + emit(Op::Sub); + return true; + }; + if (!build(0) || !build(1)) return false; + emit(Op::Shr); + if (!mask()) return false; + emit(Op::And); + return true; + } + case IR::Opcode::BitFieldSExtract: { + // (base << (32 - offset - count)) >> (32 - count), arithmetic. + if (!build(0)) return false; + emit(Op::Immediate); + code.push_back(32); + if (!build(1)) return false; + emit(Op::Sub); + if (!build(2)) return false; + emit(Op::Sub); + emit(Op::Shl); + emit(Op::Immediate); + code.push_back(32); + if (!build(2)) return false; + emit(Op::Sub); + emit(Op::Sar); + return true; + } + case IR::Opcode::BitFieldInsert: { + // (base & ~mask) | ((insert << offset) & mask), mask = ((1 << count) - 1) << offset. + const auto mask = [&] { + emit(Op::Immediate); + code.push_back(1); + if (!build(3)) return false; + emit(Op::Shl); + emit(Op::Immediate); + code.push_back(1); + emit(Op::Sub); + if (!build(2)) return false; + emit(Op::Shl); + return true; + }; + if (!build(0) || !mask()) return false; + emit(Op::Not); + emit(Op::And); + if (!build(1) || !build(2)) return false; + emit(Op::Shl); + if (!mask()) return false; + emit(Op::And); + emit(Op::Or); + return true; + } + case IR::Opcode::UClamp32: + case IR::Opcode::SClamp32: { + const bool is_signed = inst->GetOpcode() == IR::Opcode::SClamp32; + if (!build(0) || !build(1)) return false; + emit(is_signed ? Op::SMax : Op::UMax); + if (!build(2)) return false; + emit(is_signed ? Op::SMin : Op::UMin); + return true; + } + case IR::Opcode::SelectU1: + case IR::Opcode::SelectU32: + if (!build(0) || !build(1) || !build(2)) return false; + emit(Op::Select); + return true; + default: + g_handle_failure = inst->GetOpcode(); + return false; + } +} + +// Programs built while making the texture instructions of the pass running on this thread. +thread_local std::vector<std::vector<u32>>* g_handle_programs{}; + std::optional<u32> TryGetConstant(IR::Value& value, Environment& env) { const IR::Inst* inst = value.InstRecursive(); if (inst->GetOpcode() != IR::Opcode::GetCbufU32) { @@ -525,10 +911,44 @@ TextureInst MakeInst(Environment& env, IR::Block* block, IR::Inst& inst, const H if (IsBindless(inst)) { const std::optional<ConstBufferAddr> track_addr{TrackCached(inst.Arg(0), env, host_info, block)}; - if (!track_addr) { - throw NotImplementedException("Failed to track bindless texture constant buffer"); - } else { + if (track_addr) { addr = *track_addr; + } else { + // Read from a storage buffer (a handle table): computed for each draw. + std::vector<u32> code; + int budget = 4096; + g_handle_failure = IR::Opcode::Void; + if (!g_handle_programs || !BuildHandleProgram(inst.Arg(0), code, budget, block)) { + // Not computable for a whole draw (a handle that changes in a loop): keep the + // earlier search through loads, which at least gives the shader a texture. + const std::optional<ConstBufferAddr> legacy{IR::BreadthFirstSearch(inst.Arg(0), + [&env, &host_info, block](const IR::Inst* candidate) { + return TryGetConstBuffer(candidate, env, host_info, block); + }, [&](IR::Value operand) { return ResolveConditionalPhi(operand, block); })}; + if (!legacy) { + throw NotImplementedException("Failed to track bindless texture constant buffer (stopped at {}, budget {})", + g_handle_failure, budget); + } + return TextureInst{.cbuf = *legacy, .inst = &inst, .block = block}; + } + auto& programs{*g_handle_programs}; + auto it{std::ranges::find(programs, code)}; + if (it == programs.end()) { + programs.push_back(std::move(code)); + it = programs.end() - 1; + } + addr = ConstBufferAddr{ + .index = 0, + .offset = 0, + .shift_left = 0, + .secondary_index = 0, + .secondary_offset = 0, + .secondary_shift_left = 0, + .dynamic_offset = {}, + .count = 1, + .has_secondary = false, + .handle_program = static_cast<u32>(it - programs.begin()), + }; } } else { addr = ConstBufferAddr{ @@ -724,6 +1144,7 @@ void TexturePass(Environment& env, IR::Program& program, const HostTranslateInfo env.handle_cache.clear(); env.track_cache.clear(); TextureInstVector to_replace; + g_handle_programs = &program.info.handle_programs; for (IR::Block* const block : program.post_order_blocks) { env.track_cache.clear(); for (IR::Inst& inst : block->Instructions()) { @@ -881,6 +1302,7 @@ void TexturePass(Environment& env, IR::Program& program, const HostTranslateInfo .secondary_shift_left = cbuf.secondary_shift_left, .count = count, .size_shift = size_shift, + .handle_program = cbuf.handle_program, }); flags.is_integer.Assign(is_integer ? 1 : 0); } diff --git a/src/shader_recompiler/runtime_info.h b/src/shader_recompiler/runtime_info.h index e6e1284762..560afb3648 100644 --- a/src/shader_recompiler/runtime_info.h +++ b/src/shader_recompiler/runtime_info.h @@ -7,6 +7,7 @@ #pragma once #include <array> +#include <bitset> #include <map> #include <optional> #include <vector> @@ -85,6 +86,8 @@ struct TransformFeedbackVarying { struct RuntimeInfo { std::array<AttributeType, 32> generic_input_types{}; VaryingState previous_stage_stores; + /// Fragment stage after a mesh program: generic inputs it writes per primitive. + std::bitset<32> per_primitive_generics{}; std::map<IR::Attribute, IR::Attribute> previous_stage_legacy_stores_mapping; bool convert_depth_mode{}; diff --git a/src/shader_recompiler/shader_info.h b/src/shader_recompiler/shader_info.h index 71309e040e..eeb8d768d7 100644 --- a/src/shader_recompiler/shader_info.h +++ b/src/shader_recompiler/shader_info.h @@ -7,6 +7,9 @@ #pragma once #include <array> +#include <optional> +#include <span> +#include <vector> #include <bitset> #include <map> @@ -219,6 +222,9 @@ struct TextureDescriptor { u32 secondary_shift_left; u32 count; u32 size_shift; + /// Index in Info::handle_programs when the handle is computed at draw time (read from a + /// storage buffer), ~0 when it is the constant buffer word above. + u32 handle_program{~0U}; auto operator<=>(const TextureDescriptor&) const = default; }; @@ -239,11 +245,26 @@ struct ImageDescriptor { }; using ImageDescriptors = boost::container::small_vector<ImageDescriptor, 4>; +/// Switch 2 (NVN) task and mesh programs: the sizes their mesh registers set and, for mesh +/// programs, the attribute (IR::Attribute index) held by each 128-byte line of a 32-element group +/// in the emulated stage buffer: per-vertex lines from the program's output map, per-primitive +/// lines from the pass-through geometry header bound with it, in attribute order. +struct MeshInfo { + u32 payload_words{}; ///< Task output region / mesh input region, in 32-bit words + u32 max_vertices{}; + u32 max_primitives{}; + u32 topology{}; ///< SET_MESH_SHADER_A output topology (4 = triangles) + boost::container::static_vector<u32, 136> vertex_lines; + boost::container::static_vector<u32, 136> primitive_lines; +}; + struct Info { static constexpr size_t MAX_INDIRECT_CBUFS{14}; static constexpr size_t MAX_CBUFS{18}; static constexpr size_t MAX_SSBOS{32}; + bool is_task{}; ///< Switch 2 task program (Stage::Task) + bool is_mesh{}; ///< Switch 2 mesh program (Stage::Mesh) bool uses_workgroup_id{}; bool uses_local_invocation_id{}; bool uses_invocation_id{}; @@ -337,6 +358,9 @@ struct Info { TextureBufferDescriptors texture_buffer_descriptors; ImageBufferDescriptors image_buffer_descriptors; TextureDescriptors texture_descriptors; + /// Texture handles the shader reads from storage buffers, as programs over constant buffer + /// and storage buffer words (see EvaluateHandleProgram), evaluated for each draw. + std::vector<std::vector<u32>> handle_programs; ImageDescriptors image_descriptors; }; @@ -349,4 +373,111 @@ u32 NumDescriptors(const Descriptors& descriptors) { return num; } + +/// Operations of a handle program, a stack machine over 32-bit words. Operands follow the +/// operation code in the program; values are popped right operand first. +enum class HandleOp : u32 { + Immediate, ///< value + Cbuf, ///< index, offset: the constant buffer word + CbufDynamic, ///< index: the word at the popped byte offset + Storage, ///< binding: the storage buffer word at the popped byte offset + Add, Sub, Mul, And, Or, Xor, Shl, Shr, Sar, Neg, Not, + UMin, UMax, SMin, SMax, + SLt, SLe, SGt, SGe, ULt, ULe, UGt, UGe, Eq, Ne, + LAnd, LOr, LXor, LNot, + Select, ///< condition, true value, false value + Tee, ///< slot: keeps the top value in the slot (shared subexpressions) + Load, ///< slot: the value kept there +}; + +/// Runs a handle program; read_cbuf(index, offset) and read_storage(binding, offset) return the +/// words it reads. Returns nothing for a malformed program. +template <typename ReadCbuf, typename ReadStorage> +std::optional<u32> EvaluateHandleProgram(std::span<const u32> code, ReadCbuf&& read_cbuf, + ReadStorage&& read_storage) { + std::array<u32, 64> stack{}; + std::array<u32, 256> slots{}; + size_t size = 0; + const auto pop = [&]() -> u32 { return size > 0 ? stack[--size] : 0; }; + const auto push = [&](u32 value) { + if (size < stack.size()) stack[size++] = value; + }; + for (size_t pc = 0; pc < code.size();) { + const HandleOp op = static_cast<HandleOp>(code[pc++]); + const auto operand = [&]() -> u32 { return pc < code.size() ? code[pc++] : 0; }; + if (op == HandleOp::Immediate) { + push(operand()); + continue; + } + if (op == HandleOp::Cbuf) { + const u32 index = operand(); + push(read_cbuf(index, operand())); + continue; + } + if (op == HandleOp::CbufDynamic) { + const u32 index = operand(); + push(read_cbuf(index, pop())); + continue; + } + if (op == HandleOp::Storage) { + const u32 binding = operand(); + push(read_storage(binding, pop())); + continue; + } + if (op == HandleOp::Neg || op == HandleOp::Not || op == HandleOp::LNot) { + const u32 a = pop(); + push(op == HandleOp::Neg ? 0U - a : op == HandleOp::Not ? ~a : (a == 0 ? 1U : 0U)); + continue; + } + if (op == HandleOp::Tee) { + const u32 slot = operand(); + if (slot < slots.size() && size > 0) slots[slot] = stack[size - 1]; + continue; + } + if (op == HandleOp::Load) { + const u32 slot = operand(); + push(slot < slots.size() ? slots[slot] : 0U); + continue; + } + if (op == HandleOp::Select) { + const u32 false_value = pop(), true_value = pop(), condition = pop(); + push(condition != 0 ? true_value : false_value); + continue; + } + const u32 b = pop(), a = pop(); + const s32 sa = static_cast<s32>(a), sb = static_cast<s32>(b); + switch (op) { + case HandleOp::Add: push(a + b); break; + case HandleOp::Sub: push(a - b); break; + case HandleOp::Mul: push(a * b); break; + case HandleOp::And: push(a & b); break; + case HandleOp::Or: push(a | b); break; + case HandleOp::Xor: push(a ^ b); break; + case HandleOp::Shl: push(b >= 32 ? 0U : a << b); break; + case HandleOp::Shr: push(b >= 32 ? 0U : a >> b); break; + case HandleOp::Sar: push(static_cast<u32>(sa >> (b >= 32 ? 31 : b))); break; + case HandleOp::UMin: push(a < b ? a : b); break; + case HandleOp::UMax: push(a > b ? a : b); break; + case HandleOp::SMin: push(static_cast<u32>(sa < sb ? sa : sb)); break; + case HandleOp::SMax: push(static_cast<u32>(sa > sb ? sa : sb)); break; + case HandleOp::SLt: push(sa < sb); break; + case HandleOp::SLe: push(sa <= sb); break; + case HandleOp::SGt: push(sa > sb); break; + case HandleOp::SGe: push(sa >= sb); break; + case HandleOp::ULt: push(a < b); break; + case HandleOp::ULe: push(a <= b); break; + case HandleOp::UGt: push(a > b); break; + case HandleOp::UGe: push(a >= b); break; + case HandleOp::Eq: push(a == b); break; + case HandleOp::Ne: push(a != b); break; + case HandleOp::LAnd: push(a != 0 && b != 0); break; + case HandleOp::LOr: push(a != 0 || b != 0); break; + case HandleOp::LXor: push((a != 0) != (b != 0)); break; + default: return std::nullopt; + } + } + if (size != 1) return std::nullopt; + return stack[0]; +} + } // namespace Shader diff --git a/src/shader_recompiler/stage.h b/src/shader_recompiler/stage.h index 9af0a1b720..34172cfee2 100644 --- a/src/shader_recompiler/stage.h +++ b/src/shader_recompiler/stage.h @@ -17,6 +17,11 @@ enum class Stage : u32 { Compute, VertexA, + + // Switch 2 (NVN) task and mesh programs, bound in the vertex and tessellation slots while + // SET_MESH_CONTROL is enabled. + Task, + Mesh, }; constexpr u32 MaxStageTypes = 6; diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index dd6f5d97b5..a9be2ec7a8 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -17,8 +17,10 @@ add_executable(tests common/undefined_fix.cpp common/unique_function.cpp core/core_timing.cpp + core/nup_writer.cpp core/internal_network/network.cpp video_core/memory_tracker.cpp + video_core/vertex_repack.cpp video_core/ampere_returns.cpp input_common/calibration_configuration_job.cpp ) diff --git a/src/tests/core/nup_writer.cpp b/src/tests/core/nup_writer.cpp new file mode 100644 index 0000000000..ce1a0717ed --- /dev/null +++ b/src/tests/core/nup_writer.cpp @@ -0,0 +1,392 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <catch2/catch_test_macros.hpp> + +#include <array> +#include <atomic> +#include <chrono> +#include <filesystem> +#include <fstream> +#include <iterator> +#include <map> +#include <string> +#include <vector> + +#include <fmt/format.h> +#include <openssl/evp.h> + +#include "core/file_sys/nup_container.h" +#include "core/file_sys/nup_writer.h" +#include "core/file_sys/vfs/vfs_real.h" + +namespace { + +namespace fs = std::filesystem; +using FileSys::NupContainer; + +struct TempDir { + fs::path path = fs::temp_directory_path() / + ("nup-test-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + TempDir() { + fs::create_directories(path); + } + ~TempDir() { + std::error_code ec; + fs::remove_all(path, ec); + } +}; + +void Put(const fs::path& p, std::string_view value) { + fs::create_directories(p.parent_path()); + std::ofstream(p, std::ios::binary) << value; +} + +std::string ReadAll(const fs::path& p) { + std::ifstream in(p, std::ios::binary); + return {std::istreambuf_iterator<char>(in), {}}; +} + +std::string Read(const FileSys::VirtualFile& f) { + std::string out(f->GetSize(), '\0'); + f->Read(reinterpret_cast<u8*>(out.data()), out.size(), 0); + return out; +} + +// Enough of an NPDM for the packer: the program ID at ACI0 + 0x10. +std::string Npdm(u64 program_id) { + std::string npdm(0x100, '\0'); + npdm.replace(0x80, 4, "ACI0"); + for (int i = 0; i < 8; ++i) { + npdm[0x90 + i] = static_cast<char>(program_id >> (8 * i)); + } + return npdm; +} + +std::array<u8, 32> Sha256(std::string_view data) { + std::array<u8, 32> out{}; + unsigned int length{}; + EVP_Digest(data.data(), data.size(), out.data(), &length, EVP_sha256(), nullptr); + return out; +} + +u64 Le(const std::string& buf, size_t at, size_t bytes) { + u64 value = 0; + for (size_t i = 0; i < bytes; ++i) { + value |= static_cast<u64>(static_cast<u8>(buf[at + i])) << (8 * i); + } + return value; +} + +struct RawFile { + std::string name; + u64 size; + std::array<u8, 32> sha256; + u64 extent_offset; + u64 extent_size; + u8 codec; +}; + +// Reads the file records straight from the TOC, which NupContainer does not expose. +std::vector<RawFile> RawFiles(const std::string& nup) { + const u64 toc = Le(nup, 0x08, 8); + const u64 packages = Le(nup, toc + 0x04, 4); + const u64 dirs = Le(nup, toc + 0x08, 8); + const u64 files = Le(nup, toc + 0x10, 8); + const u64 pool = toc + Le(nup, toc + 0x18, 8); + std::vector<RawFile> out; + u64 pos = toc + 0x30 + packages * 0x40 + dirs * 0x18; + for (u64 i = 0; i < files; ++i, pos += 0x50) { + RawFile file{nup.substr(pool + Le(nup, pos, 4), Le(nup, pos + 4, 4)), + Le(nup, pos + 0x08, 8), + {}, + Le(nup, pos + 0x10, 8), + Le(nup, pos + 0x18, 8), + static_cast<u8>(nup[pos + 0x40])}; + std::copy_n(nup.begin() + static_cast<std::ptrdiff_t>(pos + 0x20), 32, + file.sha256.begin()); + out.push_back(file); + } + return out; +} + +// Every file under `dir` with its contents, keyed by its path relative to `dir`. +std::map<std::string, std::string> SourceTree(const fs::path& dir) { + std::map<std::string, std::string> out; + for (const auto& entry : fs::recursive_directory_iterator(dir)) { + if (entry.is_regular_file() && entry.path().filename() != ".listing") { + out[fs::relative(entry.path(), dir).generic_string()] = ReadAll(entry.path()); + } + } + return out; +} + +void PackedTree(const FileSys::VirtualDir& dir, const std::string& prefix, + std::map<std::string, std::string>& out) { + if (dir == nullptr) { + return; + } + for (const auto& file : dir->GetFiles()) { + out[prefix + file->GetName()] = Read(file); + } + for (const auto& sub : dir->GetSubdirectories()) { + PackedTree(sub, prefix + sub->GetName() + "/", out); + } +} + +std::map<std::string, std::string> PackedTree(const FileSys::VirtualDir& dir) { + std::map<std::string, std::string> out; + PackedTree(dir, "", out); + return out; +} + +constexpr u64 TitleId = 0x0400ABCD00010000ULL; + +void MakeDump(const fs::path& root) { + Put(root / "Base/ExeFS/main", "base main"); + Put(root / "Base/ExeFS/main.npdm", Npdm(TitleId)); + Put(root / "Base/RomFS/shared.bin", std::string(5000, 's')); + Put(root / "Base/RomFS/copy/shared.bin", std::string(5000, 's')); + Put(root / "Base/RomFS/empty", ""); + Put(root / "Base/RomFS/.listing", "listing"); + Put(root / "Base/RomFS/System/Resource/ResourceSizeTable.Product.100.rsizetable", "rst"); + Put(root / "Base/RomFS/big.bin", std::string(64 * 1024, 'b')); + Put(root / "Base/RomFS/random.bin", [] { + std::string data(3000, '\0'); + u32 x = 12345; + for (auto& c : data) { + x = x * 1664525 + 1013904223; + c = static_cast<char>(x >> 24); + } + return data; + }()); + Put(root / "updates/65536/exefs/main", "update main"); + Put(root / "updates/65536/exefs/main.npdm", Npdm(TitleId)); + Put(root / "updates/65536/romfs/shared.bin", std::string(5000, 's')); + Put(root / "updates/65536/romfs/System/Resource/ResourceSizeTable.Product.110.rsizetable", + "rst"); + Put(root / "updates/65536/romfs/Zeta/a", "a"); + Put(root / "updates/65536/romfs/alpha", "alpha"); + Put(root / "updates/131072/exefs/main", "update 2 main"); + Put(root / "updates/131072/exefs/main.npdm", Npdm(TitleId)); + Put(root / "dlc/0400ABCD00011001/0/romfs/addon.bin", "addon data"); +} + +} // namespace + +TEST_CASE("NUP packs base, updates and add-ons and reads back through NupContainer", "[nup]") { + TempDir t; + const auto root = t.path / "Game"; + MakeDump(root); + const auto archive = t.path / "Game.nup"; + FileSys::Nup::PackOptions options{.raw_threshold = 16 * 1024}; + std::string error; + REQUIRE(FileSys::Nup::PackDump(root, archive, options, {}, nullptr, &error) == + FileSys::Nup::PackResult::Success); + INFO(error); + REQUIRE_FALSE(fs::exists(archive.string() + ".part")); + + FileSys::RealVfsFilesystem real; + NupContainer nup{real.OpenFile(archive.string(), FileSys::OpenMode::Read)}; + REQUIRE(nup.GetStatus() == Loader::ResultStatus::Success); + REQUIRE(nup.VerifyIntegrity({}) == Loader::ResultStatus::Success); + + const auto& packages = nup.GetPackages(); + REQUIRE(packages.size() == 4); + REQUIRE(packages[0].kind == NupContainer::PackageKind::Base); + REQUIRE(packages[0].title_id == TitleId); + REQUIRE(packages[0].version == 0); + REQUIRE(packages[0].npdm.size() == 0x100); + REQUIRE(packages[1].kind == NupContainer::PackageKind::Update); + REQUIRE(packages[1].version == 65536); + REQUIRE(packages[1].title_id == TitleId); + REQUIRE_FALSE(packages[1].standalone_update); + REQUIRE(packages[2].kind == NupContainer::PackageKind::Update); + REQUIRE(packages[2].version == 131072); + REQUIRE(packages[2].romfs_dir == 0xFFFFFFFF); + REQUIRE(packages[3].kind == NupContainer::PackageKind::Addon); + REQUIRE(packages[3].title_id == 0x0400ABCD00011001ULL); + REQUIRE(packages[3].exefs_dir == 0xFFFFFFFF); + REQUIRE(nup.FindBestUpdatePackage() == 2); + + REQUIRE(PackedTree(nup.GetExeFSDir(0)) == SourceTree(root / "Base/ExeFS")); + REQUIRE(PackedTree(nup.GetRomFSDir(0)) == SourceTree(root / "Base/RomFS")); + REQUIRE(PackedTree(nup.GetExeFSDir(1)) == SourceTree(root / "updates/65536/exefs")); + REQUIRE(PackedTree(nup.GetRomFSDir(1)) == SourceTree(root / "updates/65536/romfs")); + REQUIRE(PackedTree(nup.GetExeFSDir(2)) == SourceTree(root / "updates/131072/exefs")); + REQUIRE(PackedTree(nup.GetRomFSDir(3)) == SourceTree(root / "dlc/0400ABCD00011001/0/romfs")); + REQUIRE_FALSE(nup.GetRomFSDir(0)->GetFile(".listing")); + + const auto bytes = ReadAll(archive); + const auto files = RawFiles(bytes); + std::map<std::string, std::vector<RawFile>> by_name; + for (const auto& file : files) { + by_name[file.name].push_back(file); + if (file.size != 0) { + REQUIRE(file.extent_offset % 16 == 0); + } + } + const auto shared = by_name.at("shared.bin"); + REQUIRE(shared.size() == 3); + for (const auto& file : shared) { + REQUIRE(file.sha256 == Sha256(std::string(5000, 's'))); + REQUIRE(file.codec == 1); + REQUIRE(file.extent_offset == shared[0].extent_offset); + REQUIRE(file.extent_size == shared[0].extent_size); + } + const auto& empty = by_name.at("empty").at(0); + REQUIRE(empty.size == 0); + REQUIRE(empty.extent_offset == 0); + REQUIRE(empty.extent_size == 0); + REQUIRE(empty.codec == 0); + REQUIRE(empty.sha256 == Sha256("")); + // Highly compressible, but above the raw threshold. + const auto& big = by_name.at("big.bin").at(0); + REQUIRE(big.codec == 0); + REQUIRE(big.extent_size == 64 * 1024); + REQUIRE(big.sha256 == Sha256(std::string(64 * 1024, 'b'))); + // Incompressible, so stored raw. + const auto& random = by_name.at("random.bin").at(0); + REQUIRE(random.codec == 0); + REQUIRE(random.extent_size == 3000); + REQUIRE_FALSE(by_name.contains(".listing")); +} + +TEST_CASE("NUP marks updates standalone when the dump has no base", "[nup]") { + TempDir t; + const auto root = t.path / "Game"; + Put(root / "updates/65536/exefs/main", "main"); + Put(root / "updates/65536/exefs/main.npdm", Npdm(TitleId)); + Put(root / "updates/65536/romfs/file", "rom"); + const auto archive = t.path / "Game.nup"; + REQUIRE(FileSys::Nup::PackDump(root, archive) == FileSys::Nup::PackResult::Success); + + FileSys::RealVfsFilesystem real; + NupContainer nup{real.OpenFile(archive.string(), FileSys::OpenMode::Read)}; + REQUIRE(nup.GetStatus() == Loader::ResultStatus::Success); + REQUIRE(nup.GetPackages().size() == 1); + REQUIRE(nup.GetPackages()[0].kind == NupContainer::PackageKind::Update); + REQUIRE(nup.GetPackages()[0].standalone_update); + REQUIRE(nup.GetPackages()[0].title_id == TitleId); + REQUIRE(Read(nup.GetRomFSDir(0)->GetFile("file")) == "rom"); +} + +TEST_CASE("NUP packs a bare exefs/romfs dump as one base", "[nup]") { + TempDir t; + const auto root = t.path / "Game"; + Put(root / "ExeFS/main", "main"); + Put(root / "ExeFS/main.npdm", Npdm(TitleId)); + Put(root / "RomFS/file", "rom"); + Put(root / "notes.txt", "extra"); + const auto archive = t.path / "Game.nup"; + REQUIRE(FileSys::Nup::PackDump(root, archive) == FileSys::Nup::PackResult::Success); + + FileSys::RealVfsFilesystem real; + NupContainer nup{real.OpenFile(archive.string(), FileSys::OpenMode::Read)}; + REQUIRE(nup.GetStatus() == Loader::ResultStatus::Success); + REQUIRE(nup.GetPackages().size() == 1); + REQUIRE(nup.GetPackages()[0].kind == NupContainer::PackageKind::Base); + REQUIRE_FALSE(nup.GetPackages()[0].standalone_update); + REQUIRE(Read(nup.GetExeFSDir(0)->GetFile("main")) == "main"); + REQUIRE(Read(nup.GetRomFSDir(0)->GetFile("file")) == "rom"); + const auto files = RawFiles(ReadAll(archive)); + REQUIRE(std::ranges::count(files, std::string{"notes.txt"}, &RawFile::name) == 1); +} + +TEST_CASE("NUP low storage resumes and deletes the packed sources", "[nup]") { + TempDir t; + const auto root = t.path / "Game"; + MakeDump(root); + const auto copy = t.path / "copy" / "Game"; + fs::create_directories(copy); + fs::copy(root, copy, fs::copy_options::recursive); + const auto archive = t.path / "Game.nup"; + + std::atomic_bool cancel = false; + bool cancelled = false; + FileSys::Nup::PackOptions options{.delete_sources = true, .chunk_bytes = 1}; + auto result = FileSys::Nup::PackDump( + root, archive, options, + [&](const auto& p) { + if (p.files_done >= 4 && !cancelled) { + cancel = true; + cancelled = true; + } + }, + &cancel); + REQUIRE(result == FileSys::Nup::PackResult::Cancelled); + REQUIRE(FileSys::Nup::HasResumableJournal(archive)); + REQUIRE_FALSE(fs::exists(root / "Base/ExeFS/main")); + REQUIRE_FALSE(fs::exists(root / "Base/ExeFS/main.npdm")); + + cancel = false; + std::string error; + REQUIRE(FileSys::Nup::PackDump(root, archive, options, {}, &cancel, &error) == + FileSys::Nup::PackResult::Success); + INFO(error); + REQUIRE_FALSE(fs::exists(archive.string() + ".journal")); + REQUIRE_FALSE(fs::exists(archive.string() + ".part")); + REQUIRE_FALSE(fs::exists(root / "Base/ExeFS")); + REQUIRE_FALSE(fs::exists(root / "updates")); + REQUIRE_FALSE(fs::exists(root / "dlc")); + // The listing is not packed, so it is left behind with the folders holding it. + REQUIRE(ReadAll(root / "Base/RomFS/.listing") == "listing"); + REQUIRE_FALSE(fs::exists(root / "Base/RomFS/shared.bin")); + + FileSys::RealVfsFilesystem real; + NupContainer nup{real.OpenFile(archive.string(), FileSys::OpenMode::Read)}; + REQUIRE(nup.GetStatus() == Loader::ResultStatus::Success); + REQUIRE(nup.VerifyIntegrity({}) == Loader::ResultStatus::Success); + REQUIRE(nup.GetPackages().size() == 4); + const auto npdm = Npdm(TitleId); + REQUIRE(nup.GetPackages()[0].npdm == std::vector<u8>(npdm.begin(), npdm.end())); + REQUIRE(PackedTree(nup.GetRomFSDir(0)) == SourceTree(copy / "Base/RomFS")); + REQUIRE(PackedTree(nup.GetRomFSDir(1)) == SourceTree(copy / "updates/65536/romfs")); + + // The resumed archive matches one packed in a single pass. + const auto again = t.path / "again.nup"; + REQUIRE(FileSys::Nup::PackDump(copy, again) == FileSys::Nup::PackResult::Success); + REQUIRE(ReadAll(archive) == ReadAll(again)); +} + +TEST_CASE("NUP rejects overwrite and a dump without NPDM", "[nup]") { + TempDir t; + const auto root = t.path / "Game"; + Put(root / "exefs/main", "main"); + Put(root / "romfs/a", "rom"); + const auto archive = t.path / "out.nup"; + Put(archive, "existing"); + REQUIRE(FileSys::Nup::PackDump(root, archive) == FileSys::Nup::PackResult::IoError); + REQUIRE(ReadAll(archive) == "existing"); + fs::remove(archive); + REQUIRE(FileSys::Nup::PackDump(root, archive) == FileSys::Nup::PackResult::InvalidSource); + REQUIRE_FALSE(fs::exists(archive.string() + ".part")); +} + +// Archives are meant to be byte-identical on every platform. A change to this hash means +// packing a dump no longer reproduces archives made by earlier builds (for example after a zstd +// update), so make it deliberately. +TEST_CASE("NUP output is reproducible", "[nup]") { + TempDir t; + const auto root = t.path / "Game"; + MakeDump(root); + // Large and compressible, so it covers what used to be zstd's multithreaded path. + Put(root / "Base/RomFS/large.bin", [] { + std::string data(20 * 1024 * 1024, '\0'); + u32 x = 1; + for (auto& c : data) { + x = x * 1664525 + 1013904223; + c = static_cast<char>('a' + (x >> 28)); + } + return data; + }()); + const auto archive = t.path / "Game.nup"; + REQUIRE(FileSys::Nup::PackDump(root, archive) == FileSys::Nup::PackResult::Success); + + std::string hex; + for (const u8 b : Sha256(ReadAll(archive))) { + hex += fmt::format("{:02x}", b); + } + REQUIRE(hex == "df8446903ab2c39a51af0304935eb4aa5d8ead105440976ceae7ab97133f6143"); +} diff --git a/src/tests/video_core/ampere_returns.cpp b/src/tests/video_core/ampere_returns.cpp index 56b8b374da..163758e07e 100644 --- a/src/tests/video_core/ampere_returns.cpp +++ b/src/tests/video_core/ampere_returns.cpp @@ -48,4 +48,31 @@ TEST_CASE("Ampere callees still reject writes to the link", "[video_core][ampere REQUIRE(lowered.error == "native callee overwrites link register"); REQUIRE(lowered.error_offset == 0x30); } + +TEST_CASE("Ampere callees can compare into predicates with an R0 link", "[video_core][ampere]") { + // ISETP's zero destination field is not a write to R0. + const std::array<Instruction, 5> code{{ + {{0x00007802, 0x20, 0x00000f00, 0}}, + {{0x00007944, 0x10, 0x03c00000, 0}}, + {{0x0000794d, 0, 0x03800000, 0}}, + {{0x0100720c, 0x00000002, 0x03f06070, 0}}, + {{0x00007950, 0xffffffb0, 0x03c3ffff, 0}}, + }}; + REQUIRE(static_cast<bool>(LowerToMaxwell(code))); +} + +TEST_CASE("Ampere callees reject a wide multiply over the link", "[video_core][ampere]") { + // IMAD.WIDE.U32 R54:R55 writes the R55 link. + const std::array<Instruction, 5> code{{ + {{0x00377802, 0x20, 0x00000f00, 0}}, + {{0x00007944, 0x10, 0x03c00000, 0}}, + {{0x0000794d, 0, 0x03800000, 0}}, + {{0x01367225, 0x00000002, 0x078e00ff, 0}}, + {{0x37007950, 0xffffffb0, 0x03c3ffff, 0}}, + }}; + const auto lowered = LowerToMaxwell(code); + REQUIRE(!static_cast<bool>(lowered)); + REQUIRE(lowered.error == "native callee overwrites link register"); + REQUIRE(lowered.error_offset == 0x30); +} } // namespace diff --git a/src/tests/video_core/vertex_repack.cpp b/src/tests/video_core/vertex_repack.cpp new file mode 100644 index 0000000000..6409be3ca4 --- /dev/null +++ b/src/tests/video_core/vertex_repack.cpp @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include <catch2/catch_test_macros.hpp> +#include "video_core/renderer_vulkan/vertex_repack.h" + +namespace { +using Regs = Tegra::Engines::Maxwell3D::Regs; +using Size = Regs::VertexAttribute::Size; + +Regs MakeLayout() { + Regs regs{}; + for (auto& attr : regs.vertex_attrib_format) { + attr.constant.Assign(1); + } + regs.vertex_streams[0].stride.Assign(69); + auto& position = regs.vertex_attrib_format[0]; + position.constant.Assign(0); + position.size.Assign(Size::Size_R32_G32_B32); + auto& packed = regs.vertex_attrib_format[1]; + packed.constant.Assign(0); + packed.offset.Assign(65); + packed.size.Assign(Size::Size_A2_B10_G10_R10); + return regs; +} +} + +TEST_CASE("Vertex repacking preserves aligned fields and relocates packed tail", "[vertex_repack]") { + const auto regs = MakeLayout(); + const auto mask = Vulkan::VertexRepackMask(regs); + REQUIRE(mask == 1); + REQUIRE(Vulkan::RepackedVertexOffset(mask, 0, 0) == 0); + REQUIRE(Vulkan::RepackedVertexOffset(mask, 0, 64) == 64); + REQUIRE(Vulkan::RepackedVertexOffset(mask, 0, 65) == 68); + REQUIRE(Vulkan::RepackedVertexStride(mask, 0, 69) == 72); + REQUIRE(Vulkan::RepackedVertexStride(mask, 1, 16) == 16); +} + +TEST_CASE("Vertex repacking excludes overlapping and oversized attributes", "[vertex_repack]") { + auto regs = MakeLayout(); + auto& attr = regs.vertex_attrib_format[2]; + attr.constant.Assign(0); + attr.offset.Assign(64); + attr.size.Assign(Size::Size_R32); + REQUIRE(Vulkan::VertexRepackMask(regs) == 0); + attr.offset.Assign(68); + REQUIRE(Vulkan::VertexRepackMask(regs) == 0); + attr.constant.Assign(1); + REQUIRE(Vulkan::VertexRepackMask(regs) == 1); +} + +TEST_CASE("Vertex repacking ignores constant invalid formats", "[vertex_repack]") { + auto regs = MakeLayout(); + REQUIRE(Vulkan::VertexRepackMask(regs) == 1); + regs.vertex_attrib_format[2].constant.Assign(0); + REQUIRE(Vulkan::VertexRepackMask(regs) == 0); +} + +TEST_CASE("Vertex repacking selects bindings independently", "[vertex_repack]") { + auto regs = MakeLayout(); + regs.vertex_streams[31].stride.Assign(69); + auto& attr = regs.vertex_attrib_format[2]; + attr = regs.vertex_attrib_format[1]; + attr.buffer.Assign(31); + REQUIRE(Vulkan::VertexRepackMask(regs) == 0x80000001U); + regs.vertex_streams[0].stride.Assign(72); + REQUIRE(Vulkan::VertexRepackMask(regs) == 0x80000000U); + REQUIRE(Vulkan::RepackedVertexOffset(0x80000000U, 0, 65) == 65); +} + +TEST_CASE("Vertex repacking requires the captured tail and stride", "[vertex_repack]") { + auto regs = MakeLayout(); + regs.vertex_streams[0].stride.Assign(68); + REQUIRE(Vulkan::VertexRepackMask(regs) == 0); + regs.vertex_streams[0].stride.Assign(69); + regs.vertex_attrib_format[1].offset.Assign(64); + REQUIRE(Vulkan::VertexRepackMask(regs) == 0); +} + +TEST_CASE("Static vertex repacking requires explicit experimental opt-in", "[vertex_repack]") { + REQUIRE(Vulkan::VertexRepackInputSupported(true, false)); + REQUIRE_FALSE(Vulkan::VertexRepackInputSupported(false, false)); + REQUIRE(Vulkan::VertexRepackInputSupported(false, true)); +} + +TEST_CASE("Vertex repack planning keeps descriptor offsets aligned", "[vertex_repack]") { + for (const u32 alignment : {4U, 16U, 64U, 256U, 4096U}) { + for (const u32 offset : {0U, 1U, 3U, 255U, 4099U}) { + const auto plan = Vulkan::MakeVertexRepackParameters(offset, 70, alignment, 1U << 20, 65535); + REQUIRE(plan.has_value()); + REQUIRE(plan->source_base % alignment == 0); + REQUIRE(plan->source_base + plan->source_bias == offset); + REQUIRE(plan->vertices == 2); + REQUIRE(plan->output_size == 144); + REQUIRE(plan->source_range >= plan->source_bias + 70); + REQUIRE(plan->source_range % 4 == 0); + REQUIRE(plan->dispatch_groups == 1); + } + } +} + +TEST_CASE("Vertex repack planning rejects overflow and device-limit violations", "[vertex_repack]") { + REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(0, 0, 256, 1U << 20, 65535)); + REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(0, 69, 0, 1U << 20, 65535)); + REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(0, 69, 256, 71, 65535)); + REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(255, 69, 256, 323, 65535)); + REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(0, 69, 256, 1U << 20, 0)); + REQUIRE_FALSE(Vulkan::MakeVertexRepackParameters(0, 0xffffffffU, 256, ~u64{0}, 0xffffffffU)); + const auto one = Vulkan::MakeVertexRepackParameters(0, 69, 256, 72, 1); + REQUIRE(one.has_value()); + REQUIRE(one->source_range == 72); +} diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 5ed43fc903..b3ef88c5f1 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -235,6 +235,7 @@ add_library(video_core STATIC shader_cache.h shader_environment.cpp shader_environment.h + shader_handle_program.h shader_notify.cpp shader_notify.h smaa_area_tex.h diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e90af3214d..53bbae7610 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -17,6 +17,7 @@ #include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/guest_memory.h" #include "video_core/host1x/gpu_device_memory_manager.h" +#include "video_core/nx2.h" #include "video_core/nx2_profile.h" #include "video_core/texture_cache/util.h" @@ -100,6 +101,12 @@ void BufferCache<P>::TickFrame() { return; } runtime.TickFrame(slot_buffers); + if (frame_tick % 600 == 0 && !uploaded_buffer_ranges.empty()) { + for (const auto& [address, size] : uploaded_buffer_ranges) { + device_memory.HintCold(address, size); + } + uploaded_buffer_ranges.clear(); + } if (static u64 frames{}; ++frames % 600 == 0) { u64 total{}; size_t count{}; @@ -141,9 +148,10 @@ void BufferCache<P>::TickFrame() { // title 1's buffers were ~200 MiB of an 8 GB GPU's ~6.8 GB in use, yet this collector // ran aggressively every frame on the device-wide figure, evicting up to 64 buffers (and // downloading the GPU-written ones, ~5% of the GPU thread) that were recreated later. - // Collect when the buffers themselves are large. - if (buffer_bytes >= 1_GiB) { - RunGarbageCollector(buffer_bytes >= 2_GiB); + // Collect when the buffers themselves are large. At 1 GiB it held geometry of scenes + // long gone: title 1's title screen grew to ~0.9 GB of buffers in ten minutes. + if (buffer_bytes >= 512_MiB) { + RunGarbageCollector(buffer_bytes >= 1_GiB); } } else if (total_used_memory >= minimum_memory) { RunGarbageCollector(total_used_memory >= critical_memory); @@ -291,6 +299,9 @@ bool BufferCache<P>::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am const bool has_new_downloads = tmp_intervals.size() != 0; for (const auto& pair : tmp_intervals) { gpu_modified_ranges.Add(pair.first, pair.second); + if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { + Tegra::Nx2MarkGpuWritten(pair.first, pair.second); + } } const auto& copy = copies[0]; src_buffer.MarkUsage(copy.src_offset, copy.size); @@ -379,6 +390,10 @@ template <class P> void BufferCache<P>::BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr gpu_addr, u32 size) { const std::optional<DAddr> device_addr = gpu_memory->GpuToCpuAddress(gpu_addr); + if (!device_addr) { + channel_state->uniform_buffers[stage][index] = NULL_BINDING; + return; + } const Binding binding{ .device_addr = *device_addr, .size = size, @@ -389,7 +404,11 @@ void BufferCache<P>::BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr template <class P> void BufferCache<P>::DisableGraphicsUniformBuffer(size_t stage, u32 index) { - channel_state->uniform_buffers[stage][index] = NULL_BINDING; + auto& current = channel_state->uniform_buffers[stage][index]; + if (current.device_addr == 0 && current.size == 0 && !current.buffer_id) [[likely]] { + return; + } + current = NULL_BINDING; } template <class P> @@ -994,6 +1013,26 @@ void BufferCache<P>::BindHostGraphicsUniformBuffers(size_t stage) { template <class P> void BufferCache<P>::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 binding_index, bool needs_bind) { ++channel_state->uniform_cache_shots[0]; + // Bank 24 (root table 0, NVN2's push constants) is read from bank 17, which such games do not + // bind: stream the table's current words as that buffer. + if (index == 17 && maxwell3d->root_tables.loaded && channel_state->uniform_buffers[stage][index].size == 0) { + const auto& words = maxwell3d->root_tables.tables[0]; + const u32 size = (*channel_state->uniform_buffer_sizes)[stage][index]; + const u32 copied = (std::min)(size, static_cast<u32>(sizeof(words))); + if constexpr (IS_OPENGL) { + std::array<u8, sizeof(words)> data{}; + std::memcpy(data.data(), words.data(), copied); + runtime.BindFastUniformBuffer(stage, binding_index, copied); + runtime.PushFastUniformBuffer(stage, binding_index, std::span<const u8>(data.data(), copied)); + } else { + const std::span<u8> span = runtime.BindMappedUniformBuffer(stage, binding_index, size); + std::memset(span.data(), 0, span.size()); + std::memcpy(span.data(), words.data(), copied); + } + channel_state->fast_bound_uniform_buffers[stage] |= 1u << binding_index; + channel_state->uniform_buffer_binding_sizes[stage][binding_index] = size; + return; + } const Binding& binding = channel_state->uniform_buffers[stage][index]; const DAddr device_addr = binding.device_addr; const u32 size = (std::min)(binding.size, (*channel_state->uniform_buffer_sizes)[stage][index]); @@ -1494,6 +1533,9 @@ void BufferCache<P>::UpdateDrawIndirect() { template <class P> void BufferCache<P>::UpdateUniformBuffers(size_t stage) { + if (channel_state->enabled_uniform_buffer_masks[stage] == 0) [[likely]] { + return; + } ForEachEnabledBit(channel_state->enabled_uniform_buffer_masks[stage], [&](u32 index) { Binding& binding = channel_state->uniform_buffers[stage][index]; if (binding.buffer_id) { @@ -1511,6 +1553,9 @@ void BufferCache<P>::UpdateUniformBuffers(size_t stage) { template <class P> void BufferCache<P>::UpdateStorageBuffers(size_t stage) { + if (channel_state->enabled_storage_buffers[stage] == 0) [[likely]] { + return; + } ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) { // Resolve buffer Binding& binding = channel_state->storage_buffers[stage][index]; @@ -1523,6 +1568,9 @@ void BufferCache<P>::UpdateStorageBuffers(size_t stage) { template <class P> void BufferCache<P>::UpdateTextureBuffers(size_t stage) { + if (channel_state->enabled_texture_buffers[stage] == 0) [[likely]] { + return; + } ForEachEnabledBit(channel_state->enabled_texture_buffers[stage], [&](u32 index) { Binding& binding = channel_state->texture_buffers[stage][index]; binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false); @@ -1610,14 +1658,17 @@ void BufferCache<P>::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 } else { ArmReadTraps(device_addr, size); } + if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) { + Tegra::Nx2MarkGpuWritten(device_addr, size); + } } template <class P> BufferId BufferCache<P>::FindBuffer(DAddr device_addr, u32 size, bool sparse_compatible) { - if (device_addr == 0) { + const u64 page = device_addr >> CACHING_PAGEBITS; + if (device_addr == 0 || page >= page_table.size()) { return NULL_BUFFER_ID; } - const u64 page = device_addr >> CACHING_PAGEBITS; const BufferId buffer_id = page_table[page]; if (buffer_id) { Buffer& buffer = slot_buffers[buffer_id]; @@ -1927,6 +1978,11 @@ void BufferCache<P>::MappedUploadMemory([[maybe_unused]] Buffer& buffer, u8* const src_pointer = staging_pointer.data() + copy.src_offset; const DAddr device_addr = buffer.CpuAddr() + copy.dst_offset; device_memory.ReadBlockUnsafe(device_addr, src_pointer, copy.size); + // Large uploads are mostly geometry the host GPU now has its own copy of: TickFrame + // marks their guest memory cold (see TextureCache::RefreshContents). + if (copy.size >= 256_KiB) { + uploaded_buffer_ranges.emplace_back(device_addr, copy.size); + } // Apply the staging offset copy.src_offset += upload_staging.offset; } diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index d8a61b7913..47f342932d 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -580,6 +580,8 @@ private: size_t immediate_buffer_capacity = 0; Common::ScratchBuffer<u8> immediate_buffer_alloc; + /// Guest memory large uploads came from since the last cold hint (device address, size). + std::vector<std::pair<DAddr, u64>> uploaded_buffer_ranges; struct LRUItemParams { using ObjectType = BufferId; diff --git a/src/video_core/dirty_flags.cpp b/src/video_core/dirty_flags.cpp index e6856aaca3..2f359749aa 100644 --- a/src/video_core/dirty_flags.cpp +++ b/src/video_core/dirty_flags.cpp @@ -64,6 +64,8 @@ void SetupDirtyRenderTargets(Maxwell3D::DirtyState::Tables& tables) { void SetupDirtyShaders(Maxwell3D::DirtyState::Tables& tables) { FillBlock(tables[0], OFF(pipelines), NUM(pipelines[0]) * Maxwell3D::Regs::MaxShaderProgram, Shaders); + // Task and mesh programs are translated with the mesh registers. + FillBlock(tables[1], Maxwell3D::Regs::MeshControl, 4, Shaders); } } // Anonymous namespace diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 88d571ae5f..1144488c91 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -155,13 +155,31 @@ bool DmaPusher::Step() { // (the split only starts with unsafe reads; the settings can change mid-game). const bool use_safe = !VideoCommon::GpuSplit::Get().Active() && (Settings::IsDMALevelDefault() ? Settings::IsGPULevelHigh() : Settings::IsDMALevelSafe()); - if (use_safe) { + // The words as submitted, unless this is a resumed entry or the GPU has newer ones the + // safe read below would flush (with the split, the front can't tell: it reads memory + // unflushed anyway). Entries on pages the GPU writes (culling) always take the flushing + // read: their words at submit time are an older frame's. + const bool submitted = command_resume_offset == 0 && + dma_pushbuffer_subindex < command_list.submitted_offsets.size() && + !IsGpuWrittenCommands(dma_state.dma_get, + size_t{header.size} * sizeof(u32)) && + !(use_safe && memory_manager.IsMemoryDirty(dma_state.dma_get, + header.size * sizeof(u32))); + if (submitted) { + const std::span<const CommandHeader> commands{ + command_list.submitted_words.data() + + command_list.submitted_offsets[dma_pushbuffer_subindex], + header.size}; + RecordParsedEntry(header, commands); + command_resume_offset += ProcessCommands(commands, 0); + } else if (use_safe) { Tegra::Memory::GpuGuestMemory<Tegra::CommandHeader, Tegra::Memory::GuestMemoryFlags::SafeRead>headers(memory_manager, dma_state.dma_get, header.size, &command_headers); const std::span<const CommandHeader> commands{headers.data(), headers.size()}; RecordParsedEntry(header, commands); command_resume_offset += ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset); } else { + FlushGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32)); Tegra::Memory::GpuGuestMemory<Tegra::CommandHeader, Tegra::Memory::GuestMemoryFlags::UnsafeRead>headers(memory_manager, dma_state.dma_get, header.size, &command_headers); const std::span<const CommandHeader> commands{headers.data(), headers.size()}; RecordParsedEntry(header, commands); @@ -445,6 +463,34 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) { } } +bool DmaPusher::IsGpuWrittenCommands(GPUVAddr address, size_t size) { + if (!Tegra::g_nx2_ampere.load(std::memory_order_relaxed) || size == 0) { + return false; + } + for (GPUVAddr page = address & ~GPUVAddr{0xfff}; page < address + size; page += 0x1000) { + if (const auto device = memory_manager.GpuToCpuAddress(page); + device && Tegra::Nx2IsGpuWritten(*device)) { + return true; + } + } + return false; +} + +void DmaPusher::FlushGpuWrittenCommands(GPUVAddr address, size_t size) { + // Title 1 runs small command lists its culling dispatches fill in each frame. Accurate GPU + // mode reads every command list through the caches (safe reads); here only the ones on pages + // the GPU has written are, or their draws run with an older frame's words (trees vanish). + if (!IsGpuWrittenCommands(address, size)) { + return; + } + // The caches belong to the split's back: it first runs what the front queued (the dispatch + // that writes these words among it). + VideoCommon::GpuSplit::Get().Drain(); + if (memory_manager.IsMemoryDirty(address, size, VideoCommon::CacheType::BufferCache)) { + memory_manager.FlushRegion(address, size, VideoCommon::CacheType::BufferCache); + } +} + void DmaPusher::ReportUnboundSubchannel() { // Methods sent to a subchannel with no engine (no SetObject yet) are dropped. static std::atomic<u32> reported{}; diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index 82d29edb4b..e9dd5e2b6b 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -117,6 +117,10 @@ struct CommandList final { boost::container::small_vector<CommandHeader, 512> prefetch_command_list; /// EDEN_NX2_PB_CHECK: hash of each entry's words when it was submitted. std::vector<u64> submit_hashes; + /// The entries' words when they were submitted (NX2), each entry's at submitted_offsets[i]: + /// executed instead of the memory, which the game may already be refilling for a later frame. + std::vector<CommandHeader> submitted_words; + std::vector<u32> submitted_offsets; }; /// FNV-1a over pushbuffer words (EDEN_NX2_PB_CHECK). @@ -182,6 +186,10 @@ public: void DumpCurrentEntry(const char* reason); void CheckSubmitHash(const CommandListHeader& header, u64 submit_hash); void RecordParsedEntry(const CommandListHeader& header, std::span<const CommandHeader> commands); + /// Whether the entry lies on device pages the GPU has written (NX2). + bool IsGpuWrittenCommands(GPUVAddr address, size_t size); + /// Brings command list words the GPU wrote (still only in host buffers) to guest memory. + void FlushGpuWrittenCommands(GPUVAddr address, size_t size); private: diff --git a/src/video_core/engines/draw_manager.cpp b/src/video_core/engines/draw_manager.cpp index 5891c25622..038e5a2e69 100644 --- a/src/video_core/engines/draw_manager.cpp +++ b/src/video_core/engines/draw_manager.cpp @@ -80,6 +80,33 @@ void Maxwell3D::DrawManager::DrawDeferred(Maxwell3D& maxwell3d) { draw_state.instance_count = 0; } +bool Maxwell3D::DrawManager::IsInstanceLoopMethod(u32 method) noexcept { + switch (method) { + case MAXWELL3D_REG_INDEX(draw.begin): + case MAXWELL3D_REG_INDEX(draw.end): + case MAXWELL3D_REG_INDEX(vertex_buffer.first): + case MAXWELL3D_REG_INDEX(vertex_buffer.count): + case MAXWELL3D_REG_INDEX(index_buffer.first): + case MAXWELL3D_REG_INDEX(index_buffer.count): + case MAXWELL3D_REG_INDEX(index_buffer32_first): + case MAXWELL3D_REG_INDEX(index_buffer16_first): + case MAXWELL3D_REG_INDEX(index_buffer8_first): + case MAXWELL3D_REG_INDEX(index_buffer32_subsequent): + case MAXWELL3D_REG_INDEX(index_buffer16_subsequent): + case MAXWELL3D_REG_INDEX(index_buffer8_subsequent): + case MAXWELL3D_REG_INDEX(draw_inline_index): + case MAXWELL3D_REG_INDEX(inline_index_2x16.count): + case MAXWELL3D_REG_INDEX(inline_index_2x16.even): + case MAXWELL3D_REG_INDEX(inline_index_4x8.count): + case MAXWELL3D_REG_INDEX(inline_index_4x8.index0): + case MAXWELL3D_REG_INDEX(vertex_array_instance_first): + case MAXWELL3D_REG_INDEX(vertex_array_instance_subsequent): + return true; + default: + return false; + } +} + void Maxwell3D::DrawManager::DrawArray(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology, u32 vertex_first, u32 vertex_count, u32 base_instance, u32 num_instances) { draw_state.topology = topology; draw_state.vertex_buffer.first = vertex_first; diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index ab2ba10162..4732167369 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -6,6 +6,7 @@ #include <algorithm> #include <atomic> +#include <bit> #include <bitset> #include <cstdlib> #include <fstream> @@ -557,6 +558,7 @@ bool KeplerCompute::DispatchNativeCompute(const NativeQmdScheduler::Descriptor& // The console runs this on tensor cores; stand in with a plain upscale of its input. return UpscaleInsteadOfNativeProgram(); } + RememberDlssMotionVectors(); return false; } return true; @@ -588,10 +590,38 @@ void KeplerCompute::DescribeNativeResources(const NativeQmdScheduler::Descriptor std::atomic<u32> g_nx2_trace_draws_after_upscale{}; +// Fire Emblem: Fortune's Weave's first DLSS pass (rejected, before the network) has the input colour +// at cbuf0+0x278, its motion vectors (R16G16 float, the input's size) at +0x290 and this frame's +// camera jitter (x, y; input pixels, Halton bases 2 and 3 less 0.5) at +0x1b0. +void KeplerCompute::RememberDlssMotionVectors() { + const auto& cbuf = launch_description.const_buffer_config[0]; + if (cbuf.size.Value() < 0x294) return; + const GPUVAddr cbuf_address = (GPUVAddr{cbuf.address_high} << 32) | cbuf.address_low; + const auto image = [&](u32 offset) -> std::optional<Texture::TICEntry> { + const u32 tic_index = memory_manager.Read<u32>(cbuf_address + offset) & 0xfffff; + if (tic_index > regs.tic.limit) return std::nullopt; + return GetTICEntry(tic_index); + }; + const auto color = image(0x278), motion = image(0x290); + if (!color || !motion || color->format != Texture::TextureFormat::B10G11R11 || + motion->format != Texture::TextureFormat::R16G16 || motion->Address() == 0 || + motion->Width() != color->Width() || motion->Height() != color->Height() || + !motion->IsBlockLinear()) + return; + dlss_motion_vectors = *motion; + dlss_motion_color = color->Address(); + dlss_jitter = {std::bit_cast<f32>(memory_manager.Read<u32>(cbuf_address + 0x1b0)), + std::bit_cast<f32>(memory_manager.Read<u32>(cbuf_address + 0x1b4))}; +} + // Title 4's upscaler reads the tonemapped frame through the texture handle at cbuf0+0x90 and // writes the output resolution image through the surface handle at cbuf0+0xa0 (from its -// TLD/SUST operands). Blit one to the other with bilinear filtering when both are plain 2D images -// of the same format and the output is larger. +// TLD/SUST operands). Kirby Air Riders' reads a 2048x1152 image at cbuf0+0x80 and writes a +// 2560x1440 one at +0x90; Fire Emblem: Fortune's Weave's DLSS network ends with a pass that reads +// the input colour through cbuf0+0x328 and writes the DLSS output through cbuf0+0x348 (B10G11R11; +// 720p to 1440p docked). Blit one to the other with bilinear filtering when both are plain 2D +// images of the same format and the output is larger: title 4's slots first, then Fortune's +// Weave's, else the first two handles in cbuf0+0x80..0x11c that form such a pair. bool KeplerCompute::UpscaleInsteadOfNativeProgram() { const auto& cbuf = launch_description.const_buffer_config[0]; const GPUVAddr cbuf_address = (GPUVAddr{cbuf.address_high} << 32) | cbuf.address_low; @@ -600,10 +630,37 @@ bool KeplerCompute::UpscaleInsteadOfNativeProgram() { if (tic_index > regs.tic.limit) return std::nullopt; return GetTICEntry(tic_index); }; - const auto input = image(0x90), output = image(0xa0); - if (!input || !output || input->format != output->format || input->Address() == 0 || - output->Address() == 0 || output->Width() <= input->Width() || !input->IsBlockLinear() || - !output->IsBlockLinear()) + const auto upscale_pair = [](const std::optional<Texture::TICEntry>& from, + const std::optional<Texture::TICEntry>& to) { + return from && to && from->format == to->format && from->Address() != 0 && + to->Address() != 0 && to->Width() > from->Width() && from->IsBlockLinear() && + to->IsBlockLinear(); + }; + std::optional<Texture::TICEntry> input = image(0x90), output = image(0xa0); + bool fortunes_weave = false; + if (!upscale_pair(input, output)) { + input = image(0x328); + output = image(0x348); + fortunes_weave = upscale_pair(input, output); + } + if (!upscale_pair(input, output)) { + input.reset(); + output.reset(); + for (u32 first = 0x80; first < 0x120 && !input; first += 4) { + const auto from = memory_manager.Read<u32>(cbuf_address + first) != 0 ? image(first) + : std::nullopt; + for (u32 second = first + 4; from && second < 0x120; second += 4) { + if (memory_manager.Read<u32>(cbuf_address + second) == 0) continue; + const auto to = image(second); + if (upscale_pair(from, to)) { + input = from; + output = to; + break; + } + } + } + } + if (!input || !output) return false; using RTFormat = Tegra::RenderTargetFormat; RTFormat format{}; @@ -700,7 +757,24 @@ bool KeplerCompute::UpscaleInsteadOfNativeProgram() { source.format = static_cast<RTFormat>(std::strtoul(value, nullptr, 16)); } } - const bool done = rasterizer->AccelerateSurfaceCopy(source, surface(*output), config); + // Fortune's Weave: a temporal upscale along its motion vectors that undoes this frame's camera + // jitter (both from the frame's first DLSS pass) and accumulates the frames as DLSS would. + // EDEN_NX2_TEMPORAL_UPSCALE=0 blits instead. + const auto upscale_temporally = [&] { + static const bool enabled = [] { + const char* value = std::getenv("EDEN_NX2_TEMPORAL_UPSCALE"); + return value == nullptr || std::string_view{value} != "0"; + }(); + if (!fortunes_weave || !enabled || !dlss_motion_vectors || + dlss_motion_color != input->Address()) + return false; + Fermi2D::Surface motion = surface(*dlss_motion_vectors); + motion.format = RTFormat::R16G16_FLOAT; + return rasterizer->AccelerateTemporalUpscale(source, surface(*output), motion, dlss_jitter, + {1.0f, 1.0f}); + }; + const bool done = upscale_temporally() || + rasterizer->AccelerateSurfaceCopy(source, surface(*output), config); if (static u32 checked{}; checked++ == sample_at) { const u64 size = u64{output->Width()} * output->Height() * 4; if (const auto device = memory_manager.GpuToCpuAddress(output->Address())) { diff --git a/src/video_core/engines/kepler_compute.h b/src/video_core/engines/kepler_compute.h index b525636dde..b5aee3706a 100644 --- a/src/video_core/engines/kepler_compute.h +++ b/src/video_core/engines/kepler_compute.h @@ -266,7 +266,15 @@ private: bool DispatchNativeCompute(const NativeQmdScheduler::Descriptor& descriptor); void ProcessNativeReportSemaphore(u32 argument); void DescribeNativeResources(const NativeQmdScheduler::Descriptor& qmd); + void RememberDlssMotionVectors(); bool UpscaleInsteadOfNativeProgram(); + + /// Motion vectors of Fire Emblem: Fortune's Weave's DLSS evaluation, from its first pass (run + /// earlier in the frame than the network pass the stand-in replaces), with the input colour + /// image and camera jitter they go with. + std::optional<Texture::TICEntry> dlss_motion_vectors; + GPUVAddr dlss_motion_color{}; + std::array<f32, 2> dlss_jitter{}; bool ProcessNativeMethod(Core::System& system, u32 method, u32 argument, bool is_last_call); NativeQmdScheduler native_qmd; std::array<u32, 0xc00>* native_data_ram{}; diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 6adfcf89b8..9731f54584 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -164,6 +164,8 @@ bool Maxwell3D::IsMethodExecutable(u32 method) { case MAXWELL3D_REG_INDEX(vertex_array_instance_first): case MAXWELL3D_REG_INDEX(vertex_array_instance_subsequent): case MAXWELL3D_REG_INDEX(draw_texture.src_y0): + case RootTables::SelectMethod: + case RootTables::LoadMethod: case MAXWELL3D_REG_INDEX(wait_for_idle): case MAXWELL3D_REG_INDEX(shadow_ram_control): case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr): @@ -336,6 +338,12 @@ void Maxwell3D::ConsumeSinkImpl(Core::System& system) { } void Maxwell3D::ProcessDirtyRegisters(u32 method, u32 argument) { + // The instances after the first are drawn together once their loop ends, and they must see + // the state they were begun with: NVN's draw macros reset the base instance (and its copy in + // c[0][0x644]) right after the loop, before the macro returns. + if (draw_manager.HasDeferredDraw() && !DrawManager::IsInstanceLoopMethod(method)) [[unlikely]] { + draw_manager.DrawDeferred(*this); + } regs.reg_array[method] = argument; for (auto const& table : dirty.tables) dirty.flags[table[method]] = true; @@ -346,6 +354,17 @@ void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argume LOG_DEBUG(HW_GPU, "Native GPU surface clear: value={:#x}", argument); } switch (method) { + case RootTables::SelectMethod: + root_tables.selected = argument & 7; + root_tables.offset = (argument >> 8) & 0xff; + return; + case RootTables::LoadMethod: + if (root_tables.offset < 256 && root_tables.offset % 4 == 0) { + root_tables.tables[root_tables.selected][root_tables.offset / 4] = argument; + root_tables.loaded = true; + } + root_tables.offset += 4; + return; case MAXWELL3D_REG_INDEX(wait_for_idle): return rasterizer->WaitForIdle(); case MAXWELL3D_REG_INDEX(shadow_ram_control): @@ -516,6 +535,10 @@ void Maxwell3D::CallMultiMethod(Core::System& system, u32 method, const u32* bas ProcessMacro(system, method, base_start, amount, amount == methods_pending); return; } + // As in ProcessDirtyRegisters: the paths below write without it. + if (draw_manager.HasDeferredDraw() && !DrawManager::IsInstanceLoopMethod(method)) [[unlikely]] { + draw_manager.DrawDeferred(*this); + } switch (method) { case MAXWELL3D_REG_INDEX(const_buffer.buffer): case MAXWELL3D_REG_INDEX(const_buffer.buffer) + 1: @@ -819,6 +842,8 @@ bool FrontHandledOutsideDrawManager(u32 method) { return true; } switch (method) { + case Maxwell3D::RootTables::SelectMethod: + case Maxwell3D::RootTables::LoadMethod: case MAXWELL3D_REG_INDEX(wait_for_idle): case MAXWELL3D_REG_INDEX(shadow_ram_control): case MAXWELL3D_REG_INDEX(load_mme.instruction_ptr): diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 0121f1a01c..af0e37978e 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -2541,6 +2541,19 @@ public: return IsShaderConfigEnabled(static_cast<std::size_t>(type)); } + // Switch 2 (NVN) task and mesh pipelines (clc597.h): SET_MESH_CONTROL, SET_MESH_SHADER_A + // and _B, SET_MESH_INIT_SHADER. While mesh control is on, the vertex slot holds the task + // program (or the mesh program when there is no task), the tessellation slot the mesh + // program and the geometry slot a pass-through header naming the per-primitive outputs. + static constexpr std::size_t MeshControl = 0x114c / 4; + static constexpr std::size_t MeshShaderA = 0x1150 / 4; + static constexpr std::size_t MeshShaderB = 0x1154 / 4; + static constexpr std::size_t MeshInitShader = 0x1158 / 4; + + bool IsMeshEnabled() const { + return (reg_array[MeshControl] & 1) != 0; + } + struct ConstantBuffer { u32 size; u32 address_high; @@ -3083,6 +3096,12 @@ public: void ProcessMethodCall(Maxwell3D& maxwell3d, u32 method, u32 argument); void Clear(Maxwell3D& maxwell3d, u32 layer_count); void DrawDeferred(Maxwell3D& maxwell3d); + /// Instances after the first wait to be drawn together (DrawDeferred). + [[nodiscard]] bool HasDeferredDraw() const noexcept { + return draw_state.draw_mode == DrawMode::Instance && draw_state.instance_count != 0; + } + /// Methods an instance loop writes between its draws; any other write ends the loop. + [[nodiscard]] static bool IsInstanceLoopMethod(u32 method) noexcept; void DrawArray(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology, u32 vertex_first, u32 vertex_count, u32 base_instance, u32 num_instances); void DrawArrayInstanced(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology, u32 vertex_first, u32 vertex_count, bool subsequent); void DrawIndex(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology, u32 index_first, u32 index_count, u32 base_index, u32 base_instance, u32 num_instances); @@ -3160,6 +3179,18 @@ public: std::array<u32, 0xc00>& GetNativeDataRam() { return native_data_ram; } std::span<u32> GetNativeMacroPositions() { return native_macro_context ? std::span<u32>{native_macro_context->positions} : std::span<u32>{macro_positions}; } + /// Root tables: eight 256-byte tables written inline (method 0x504 selects a table, bits 0-2, + /// and a byte offset, bits 8-15; each 0x508 writes the next word) that shaders read as constant + /// banks 24-31. NVN2 keeps push constants in bank 24, which the Ampere lowering reads as bank 17. + struct RootTables { + static constexpr u32 SelectMethod = 0x504 / 4; + static constexpr u32 LoadMethod = 0x508 / 4; + std::array<std::array<u32, 64>, 8> tables{}; + u32 selected{}; + u32 offset{}; + bool loaded{}; + } root_tables; + /// Write the value to the register identified by method. void CallMethod(Core::System& system, u32 method, u32 method_argument, bool is_last_call) override; diff --git a/src/video_core/engines/sw_blitter/blitter.cpp b/src/video_core/engines/sw_blitter/blitter.cpp index 8bcc2f7a76..78363bed11 100644 --- a/src/video_core/engines/sw_blitter/blitter.cpp +++ b/src/video_core/engines/sw_blitter/blitter.cpp @@ -5,6 +5,8 @@ #include <cmath> #include <vector> +#include "common/alignment.h" +#include "common/literals.h" #include "common/scratch_buffer.h" #include "video_core/engines/sw_blitter/blitter.h" #include "video_core/engines/sw_blitter/converter.h" @@ -23,6 +25,7 @@ using VideoCore::Surface::PixelFormatFromRenderTargetFormat; namespace Tegra::Engines::Blitter { using namespace Texture; +using namespace Common::Literals; namespace { @@ -125,6 +128,46 @@ void ProcessPitchLinear(std::span<const u8> input, std::span<u8> output, size_t } } +struct SubSurface { + u64 offset; ///< From the surface's address + size_t size; ///< Bytes + u32 height; ///< Lines the part has + u32 origin_y; ///< The blit's first line in it +}; + +// The part of a surface a blit covers: the rows of blocks (block linear) or the lines (pitch) from +// its first line to its last. A tile copied into a layer of a large texture used to read and write +// back the whole layer (64-256 MiB), which also gave every block of a Switch 2 sparse layer memory. +SubSurface CoveredRows(const Fermi2D::Surface& surface, u32 bytes_per_pixel, s32 x0, s32 y0, + u32 extent_x, u32 extent_y, size_t surface_size) { + const SubSurface whole{0, surface_size, surface.height, static_cast<u32>(y0)}; + if (x0 < 0 || y0 < 0 || extent_y == 0) { + return whole; + } + const u64 first_line = static_cast<u32>(y0); + if (surface.linear == Fermi2D::MemoryLayout::BlockLinear) { + if (surface.depth != 1) { + return whole; + } + const u32 block_height = surface.block_height; + const u32 block_depth = surface.block_depth; + const u32 lines_shift = GOB_SIZE_Y_SHIFT + block_height; + const u64 row_size = + u64{Common::AlignUpLog2(surface.width * bytes_per_pixel, GOB_SIZE_X_SHIFT)} + << (lines_shift + block_depth); + const u64 first_row = first_line >> lines_shift; + const u64 rows = ((first_line + extent_y - 1) >> lines_shift) - first_row + 1; + const SubSurface part{first_row * row_size, static_cast<size_t>(rows * row_size), + static_cast<u32>(rows << lines_shift), + static_cast<u32>(first_line - (first_row << lines_shift))}; + return part.offset + part.size <= surface_size ? part : whole; + } + const u64 size = (extent_y - 1) * u64{surface.pitch} + + (static_cast<u64>(x0) + extent_x) * bytes_per_pixel; + const SubSurface part{first_line * surface.pitch, static_cast<size_t>(size), extent_y, 0}; + return part.offset + part.size <= surface_size ? part : whole; +} + } // namespace struct SoftwareBlitEngine::BlitEngineImpl { @@ -133,6 +176,23 @@ struct SoftwareBlitEngine::BlitEngineImpl { Common::ScratchBuffer<u8> dst_buffer; Common::ScratchBuffer<f32> intermediate_src; Common::ScratchBuffer<f32> intermediate_dst; + + // Buffers grown for one large surface are freed after the blit rather than kept for the rest + // of the session; small ones stay. 16 MiB kept title 1's boot mip blits (64/16/4/1 MiB levels) + // in 16 such buffers, 256 MiB, across its 2D engines. + void ReleaseLargeBuffers() { + constexpr size_t LargeCapacity = 2_MiB; + for (auto* const buffer : {&tmp_buffer, &src_buffer, &dst_buffer}) { + if (buffer->capacity() > LargeCapacity) { + buffer->release(); + } + } + for (auto* const buffer : {&intermediate_src, &intermediate_dst}) { + if (buffer->capacity() * sizeof(f32) > LargeCapacity) { + buffer->release(); + } + } + } ConverterFactory converter_factory; }; @@ -161,9 +221,11 @@ bool SoftwareBlitEngine::Blit(Fermi2D::Surface& src, Fermi2D::Surface& dst, const auto src_bytes_per_pixel = BytesPerBlock(PixelFormatFromRenderTargetFormat(src.format)); const auto dst_bytes_per_pixel = BytesPerBlock(PixelFormatFromRenderTargetFormat(dst.format)); const size_t src_size = get_surface_size(src, src_bytes_per_pixel); + const SubSurface src_part = CoveredRows(src, src_bytes_per_pixel, config.src_x0, config.src_y0, + src_extent_x, src_extent_y, src_size); Tegra::Memory::GpuGuestMemory<u8, Tegra::Memory::GuestMemoryFlags::SafeRead> tmp_buffer( - memory_manager, src.Address(), src_size, &impl->tmp_buffer); + memory_manager, src.Address() + src_part.offset, src_part.size, &impl->tmp_buffer); const size_t src_copy_size = src_extent_x * src_extent_y * src_bytes_per_pixel; const size_t dst_copy_size = dst_extent_x * dst_extent_y * dst_bytes_per_pixel; @@ -202,12 +264,13 @@ bool SoftwareBlitEngine::Blit(Fermi2D::Surface& src, Fermi2D::Surface& dst, impl->dst_buffer.resize_destructive(dst_copy_size); if (src.linear == Fermi2D::MemoryLayout::BlockLinear) { - UnswizzleSubrect(impl->src_buffer, tmp_buffer, src_bytes_per_pixel, src.width, src.height, - src.depth, config.src_x0, config.src_y0, src_extent_x, src_extent_y, - src.block_height, src.block_depth, src_extent_x * src_bytes_per_pixel); + UnswizzleSubrect(impl->src_buffer, tmp_buffer, src_bytes_per_pixel, src.width, + src_part.height, src.depth, config.src_x0, src_part.origin_y, + src_extent_x, src_extent_y, src.block_height, src.block_depth, + src_extent_x * src_bytes_per_pixel); } else { ProcessPitchLinear<false>(tmp_buffer, impl->src_buffer, src_extent_x, src_extent_y, - src.pitch, config.src_x0, config.src_y0, src_bytes_per_pixel); + src.pitch, config.src_x0, src_part.origin_y, src_bytes_per_pixel); } // Conversion Phase @@ -222,18 +285,25 @@ bool SoftwareBlitEngine::Blit(Fermi2D::Surface& src, Fermi2D::Surface& dst, } const size_t dst_size = get_surface_size(dst, dst_bytes_per_pixel); - Tegra::Memory::GpuGuestMemoryScoped<u8, Tegra::Memory::GuestMemoryFlags::SafeReadWrite> - tmp_buffer2(memory_manager, dst.Address(), dst_size, &impl->tmp_buffer); - - if (dst.linear == Fermi2D::MemoryLayout::BlockLinear) { - SwizzleSubrect(tmp_buffer2, impl->dst_buffer, dst_bytes_per_pixel, dst.width, dst.height, - dst.depth, config.dst_x0, config.dst_y0, dst_extent_x, dst_extent_y, - dst.block_height, dst.block_depth, dst_extent_x * dst_bytes_per_pixel); - } else { - ProcessPitchLinear<true>(impl->dst_buffer, tmp_buffer2, dst_extent_x, dst_extent_y, - dst.pitch, config.dst_x0, config.dst_y0, - static_cast<size_t>(dst_bytes_per_pixel)); - } + const SubSurface dst_part = CoveredRows(dst, dst_bytes_per_pixel, config.dst_x0, config.dst_y0, + dst_extent_x, dst_extent_y, dst_size); + { + Tegra::Memory::GpuGuestMemoryScoped<u8, Tegra::Memory::GuestMemoryFlags::SafeReadWrite> + tmp_buffer2(memory_manager, dst.Address() + dst_part.offset, dst_part.size, + &impl->tmp_buffer); + + if (dst.linear == Fermi2D::MemoryLayout::BlockLinear) { + SwizzleSubrect(tmp_buffer2, impl->dst_buffer, dst_bytes_per_pixel, dst.width, + dst_part.height, dst.depth, config.dst_x0, dst_part.origin_y, + dst_extent_x, dst_extent_y, dst.block_height, dst.block_depth, + dst_extent_x * dst_bytes_per_pixel); + } else { + ProcessPitchLinear<true>(impl->dst_buffer, tmp_buffer2, dst_extent_x, dst_extent_y, + dst.pitch, config.dst_x0, dst_part.origin_y, + static_cast<size_t>(dst_bytes_per_pixel)); + } + } // tmp_buffer2 writes the destination back here + impl->ReleaseLargeBuffers(); return true; } diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h index 61c9844cf2..d9f0ac2837 100644 --- a/src/video_core/fence_manager.h +++ b/src/video_core/fence_manager.h @@ -18,6 +18,7 @@ #include <queue> #include "common/common_types.h" +#include "common/logging.h" #include "common/settings.h" #include "common/thread.h" #include "video_core/delayed_destruction_ring.h" @@ -123,6 +124,13 @@ public: } void SignalSyncPoint(u32 value) { + // The id comes from a command list (the 3D engine's sync method, the puller's fence + // action). Past the syncpoint table, the increment would walk memory after it as an + // action list; command words read wrong carry any id (Splatoon Raiders on Linux). + if (value >= Tegra::Host1x::SyncpointManager::NUM_MAX_SYNCPOINTS) { + LOG_ERROR(HW_GPU, "Signal of syncpoint {} past the table, ignored", value); + return; + } syncpoint_manager.IncrementGuest(value); std::function<void()> func([this, value] { syncpoint_manager.IncrementHost(value); }); SignalFence(std::move(func)); diff --git a/src/video_core/host1x/syncpoint_manager.h b/src/video_core/host1x/syncpoint_manager.h index 847ed20c8d..d6e65af3eb 100644 --- a/src/video_core/host1x/syncpoint_manager.h +++ b/src/video_core/host1x/syncpoint_manager.h @@ -18,6 +18,8 @@ namespace Host1x { class SyncpointManager { public: + static constexpr size_t NUM_MAX_SYNCPOINTS = 192; + u32 GetGuestSyncpointValue(u32 id) const { return syncpoints_guest[id].load(std::memory_order_acquire); } @@ -78,8 +80,6 @@ private: void Wait(std::atomic<u32>& syncpoint, std::condition_variable& wait_cv, u32 expected_value); - static constexpr size_t NUM_MAX_SYNCPOINTS = 192; - std::array<std::atomic<u32>, NUM_MAX_SYNCPOINTS> syncpoints_guest{}; std::array<std::atomic<u32>, NUM_MAX_SYNCPOINTS> syncpoints_host{}; diff --git a/src/video_core/host_shaders/CMakeLists.txt b/src/video_core/host_shaders/CMakeLists.txt index 35b350c2cd..9ff7563ff7 100644 --- a/src/video_core/host_shaders/CMakeLists.txt +++ b/src/video_core/host_shaders/CMakeLists.txt @@ -14,6 +14,7 @@ set(GLSL_INCLUDES ) set(SHADER_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_vertex_repack.comp ${CMAKE_CURRENT_SOURCE_DIR}/astc_decoder.comp ${CMAKE_CURRENT_SOURCE_DIR}/bc_encoder.comp ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_float.frag @@ -71,6 +72,7 @@ set(SHADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/smaa_blending_weight_calculation.frag ${CMAKE_CURRENT_SOURCE_DIR}/smaa_neighborhood_blending.vert ${CMAKE_CURRENT_SOURCE_DIR}/smaa_neighborhood_blending.frag + ${CMAKE_CURRENT_SOURCE_DIR}/temporal_upscale.frag ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_blit_depth_stencil.frag ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_color_clear.frag ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_color_clear.vert diff --git a/src/video_core/host_shaders/temporal_upscale.frag b/src/video_core/host_shaders/temporal_upscale.frag new file mode 100644 index 0000000000..0e76df2b43 --- /dev/null +++ b/src/video_core/host_shaders/temporal_upscale.frag @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// Stands in for a game's DLSS evaluation (the tensor-core network cannot run here): a temporal +// upscale. Each frame's input samples are placed where the camera jitter put them and weighted by +// their distance to the output pixel; they accumulate with the previous output, reprojected along +// the game's motion vectors (bicubic) and clipped to the current 3x3 neighbourhood. + +#version 450 + +layout(binding = 0) uniform sampler2D input_color; +layout(binding = 1) uniform sampler2D motion_vectors; +layout(binding = 2) uniform sampler2D history; + +layout(push_constant) uniform PushConstants { + // The vertex stage's texture scale and offset come first. + layout(offset = 16) vec2 jitter; // this frame's sample offset, in input pixels + vec2 motion_scale; // motion vector -> UV offset to the previous frame + float current_weight; // largest weight of this frame; 1 ignores the history +}; + +layout(location = 0) in vec2 texcoord; // 0 to 1 across the output +layout(location = 0) out vec4 color; + +// The blit helper's samplers take unnormalized (texel) coordinates. + +// Blending in a range-compressed space keeps bright pixels from dominating the average. +vec3 Compress(vec3 c) { + return c / (1.0 + max(max(c.r, c.g), c.b)); +} + +vec3 Expand(vec3 c) { + c = clamp(c, vec3(0.0), vec3(1.0 - 1.0 / 65504.0)); + return c / (1.0 - max(max(c.r, c.g), c.b)); +} + +vec3 RgbToYCoCg(vec3 c) { + return vec3(dot(c, vec3(0.25, 0.5, 0.25)), dot(c, vec3(0.5, 0.0, -0.5)), + dot(c, vec3(-0.25, 0.5, -0.25))); +} + +vec3 YCoCgToRgb(vec3 c) { + return vec3(c.x + c.y - c.z, c.x + c.z, c.x - c.y - c.z); +} + +// Moves value towards the box centre until it lies inside the box. +vec3 ClipToBox(vec3 value, vec3 low, vec3 high) { + const vec3 center = 0.5 * (low + high); + const vec3 extent = 0.5 * (high - low) + 1.0 / 65504.0; + const vec3 offset = value - center; + const vec3 units = abs(offset / extent); + const float largest = max(units.x, max(units.y, units.z)); + return largest > 1.0 ? center + offset / largest : value; +} + +// Catmull-Rom filtered history from five bilinear fetches: repeated bilinear reprojection would +// blur the accumulated image a little more every frame. +vec3 SampleHistory(vec2 position) { + const vec2 texel1 = floor(position - 0.5) + 0.5; + const vec2 f = position - texel1; + const vec2 w0 = f * (-0.5 + f * (1.0 - 0.5 * f)); + const vec2 w1 = 1.0 + f * f * (-2.5 + 1.5 * f); + const vec2 w2 = f * (0.5 + f * (2.0 - 1.5 * f)); + const vec2 w3 = f * f * (-0.5 + 0.5 * f); + const vec2 w12 = w1 + w2; + const vec2 texel0 = texel1 - 1.0; + const vec2 texel3 = texel1 + 2.0; + const vec2 texel12 = texel1 + w2 / w12; + vec3 sum = texture(history, vec2(texel12.x, texel0.y)).rgb * (w12.x * w0.y); + sum += texture(history, vec2(texel0.x, texel12.y)).rgb * (w0.x * w12.y); + sum += texture(history, texel12).rgb * (w12.x * w12.y); + sum += texture(history, vec2(texel3.x, texel12.y)).rgb * (w3.x * w12.y); + sum += texture(history, vec2(texel12.x, texel3.y)).rgb * (w12.x * w3.y); + const float weight = w12.x * w0.y + w0.x * w12.y + w12.x * w12.y + w3.x * w12.y + w12.x * w3.y; + return max(sum / weight, vec3(0.0)); +} + +void main() { + const ivec2 input_size = textureSize(input_color, 0); + const ivec2 motion_size = textureSize(motion_vectors, 0); + // The output pixel's centre in input pixels. Input texel k sampled the scene at k + 0.5 - jitter + // (the sign measured against consecutive frames). + const vec2 target = texcoord * vec2(input_size); + const ivec2 nearest = ivec2(floor(target + jitter)); + + vec3 filtered = vec3(0.0); + float total_weight = 0.0; + float nearest_weight = 0.0; + vec3 sum = vec3(0.0); + vec3 squares = vec3(0.0); + vec2 motion = vec2(0.0); + float longest = -1.0; + for (int y = -1; y <= 1; ++y) { + for (int x = -1; x <= 1; ++x) { + const ivec2 texel = clamp(nearest + ivec2(x, y), ivec2(0), input_size - 1); + const vec3 sample_color = RgbToYCoCg(Compress(texelFetch(input_color, texel, 0).rgb)); + // A Blackman-Harris-like window over the sample's distance to the output pixel. + const vec2 distance = vec2(texel) + 0.5 - jitter - target; + const float weight = exp(-2.29 * dot(distance, distance)); + filtered += sample_color * weight; + total_weight += weight; + nearest_weight = max(nearest_weight, weight); + sum += sample_color; + squares += sample_color * sample_color; + // The longest motion around the pixel, so that an edge follows the object in front. + const vec2 candidate = + texelFetch(motion_vectors, clamp(texel, ivec2(0), motion_size - 1), 0).rg; + const float length_squared = dot(candidate, candidate); + if (length_squared > longest) { + longest = length_squared; + motion = candidate; + } + } + } + const vec3 current = filtered / total_weight; + const vec3 mean = sum / 9.0; + const vec3 deviation = sqrt(max(squares / 9.0 - mean * mean, vec3(0.0))); + const vec3 low = mean - deviation; + const vec3 high = mean + deviation; + + const vec2 previous = texcoord + motion * motion_scale; + // A frame whose samples land close to this pixel counts more; one whose samples land between + // pixels mostly keeps the history (from other jitter positions) there. + float weight = current_weight * nearest_weight; + if (current_weight >= 1.0 || any(lessThan(previous, vec2(0.0))) || + any(greaterThan(previous, vec2(1.0)))) { + weight = 1.0; + } + const vec2 history_position = previous * vec2(textureSize(history, 0)); + const vec3 past = ClipToBox(RgbToYCoCg(Compress(SampleHistory(history_position))), low, high); + color = vec4(Expand(YCoCgToRgb(mix(past, current, weight))), 1.0); +} diff --git a/src/video_core/host_shaders/vulkan_vertex_repack.comp b/src/video_core/host_shaders/vulkan_vertex_repack.comp new file mode 100644 index 0000000000..de9c649d93 --- /dev/null +++ b/src/video_core/host_shaders/vulkan_vertex_repack.comp @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#version 450 +layout(local_size_x = 128) in; +layout(std430, set = 0, binding = 0) readonly buffer Source { uint source_words[]; }; +layout(std430, set = 0, binding = 1) writeonly buffer Destination { uint dest_words[]; }; +layout(push_constant) uniform Parameters { uint source_bias; uint source_size; uint total_words; }; + +// Diagnostic conversion of the captured 69-byte layout: insert three bytes before offset 65. +void main() { + uint word = gl_GlobalInvocationID.x; + if (word >= total_words) return; + uint vertex = word / 18; + uint offset = (word % 18) * 4; + uint value = 0; + for (uint i = 0; i < 4; ++i) { + uint dest_byte = offset + i; + if (dest_byte >= 65 && dest_byte < 68) continue; + uint src_byte = vertex * 69 + dest_byte - (dest_byte >= 68 ? 3 : 0); + if (src_byte >= source_size) continue; + uint address = source_bias + src_byte; + uint byte_value = (source_words[address / 4] >> ((address % 4) * 8)) & 255; + value |= byte_value << (i * 8); + } + dest_words[word] = value; +} diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 9111ba10d2..ddfd2529fd 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -302,7 +302,7 @@ void MemoryManager::Write(GPUVAddr addr, T data) { return; } // A Switch 2 sparse block nobody wrote yet gets its memory now. - Tegra::Nx2BackSparseWrite(addr, sizeof(T)); + Tegra::Nx2BackSparseWrite(addr, sizeof(T), &data); if (auto page_pointer{GetPointer(addr)}; page_pointer) { std::memcpy(page_pointer, &data, sizeof(T)); return; @@ -431,11 +431,14 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std: if (!unsafe) { rasterizer->FlushRegion(dev_addr_base, copy_amount, which); } - if (!IsBigPageContinuous(page_index)) [[unlikely]] { - memory.ReadBlockUnsafe(dev_addr_base, dest_buffer, copy_amount); - } else { - u8* physical = memory.GetPointer<u8>(dev_addr_base); + // A continuous big page may still have no host memory (a Switch 2 region backed as it is + // written, while its backing is pending): the per-page copy handles that. + u8* const physical = IsBigPageContinuous(page_index) ? memory.GetPointer<u8>(dev_addr_base) + : nullptr; + if (physical != nullptr) [[likely]] { std::memcpy(dest_buffer, physical, copy_amount); + } else { + memory.ReadBlockUnsafe(dev_addr_base, dest_buffer, copy_amount); } dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount; }; @@ -466,7 +469,7 @@ void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer, co void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, [[maybe_unused]] VideoCommon::CacheType which, bool unsafe) { // Give Switch 2 sparse pages the write lands on their memory first (otherwise it is dropped). - Tegra::Nx2BackSparseWrite(gpu_dest_addr, size); + Tegra::Nx2BackSparseWrite(gpu_dest_addr, size, src_buffer); auto just_advance = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, std::size_t copy_amount) { src_buffer = static_cast<const u8*>(src_buffer) + copy_amount; }; @@ -485,11 +488,12 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe if (!unsafe) { rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which); } - if (!IsBigPageContinuous(page_index)) [[unlikely]] { - memory.WriteBlockUnsafe(dev_addr_base, src_buffer, copy_amount); - } else { - u8* physical = memory.GetPointer<u8>(dev_addr_base); + u8* const physical = IsBigPageContinuous(page_index) ? memory.GetPointer<u8>(dev_addr_base) + : nullptr; + if (physical != nullptr) [[likely]] { std::memcpy(physical, src_buffer, copy_amount); + } else { + memory.WriteBlockUnsafe(dev_addr_base, src_buffer, copy_amount); } src_buffer = static_cast<const u8*>(src_buffer) + copy_amount; }; diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index 318607384b..db8c92adfd 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -20,6 +20,44 @@ namespace Tegra { // Ampere classes and the Turing+ macro ISA, which Eden's Maxwell engines don't implement. inline std::atomic<bool> g_nx2_ampere{false}; +// Device pages the GPU has written through the buffer cache (shader stores, copies), ever. Title 1 +// runs command lists the GPU writes each frame (20-byte entries its culling fills in); read +// without a flush they are stale and the draws behind them (trees) go missing. The bits never +// clear: the dispatch that rewrites a page may still be queued on the split's back when its front +// reads the command list. One bit per 4 KiB of the 36-bit device address space (2 MiB). +inline constexpr u64 Nx2WrittenPageBits = 12; +inline std::array<std::atomic<u64>, (u64{1} << (36 - Nx2WrittenPageBits)) / 64> g_nx2_gpu_written_pages{}; + +inline void Nx2MarkGpuWritten(u64 device_address, u64 size) { + if (size == 0) { + return; + } + const u64 last = std::min<u64>((device_address + size - 1) >> Nx2WrittenPageBits, + g_nx2_gpu_written_pages.size() * 64 - 1); + for (u64 page = device_address >> Nx2WrittenPageBits; page <= last; ++page) { + auto& word = g_nx2_gpu_written_pages[page / 64]; + const u64 bit = u64{1} << (page % 64); + if ((word.load(std::memory_order_relaxed) & bit) == 0) { + word.fetch_or(bit, std::memory_order_relaxed); + } + } +} + +/// A new emulation session starts: no page has been written by its GPU yet. +inline void Nx2ClearGpuWrittenPages() { + for (auto& word : g_nx2_gpu_written_pages) { + word.store(0, std::memory_order_relaxed); + } +} + +[[nodiscard]] inline bool Nx2IsGpuWritten(u64 device_address) { + const u64 page = device_address >> Nx2WrittenPageBits; + if (page >= g_nx2_gpu_written_pages.size() * 64) { + return false; + } + return (g_nx2_gpu_written_pages[page / 64].load(std::memory_order_relaxed) >> (page % 64)) & 1; +} + // NVN2 queues release their progress values into the second page of a two-page GPU-accessible // region and GPU acquires read the same offsets in the first page (the CPU polls the second). // Something on the console copies one into the other; until that is known, the puller mirrors @@ -59,8 +97,9 @@ inline void Nx2ClearSemaphoreRegions() { // Switch 2 sparse regions (nx2:gpu 214, word 7 bit 0x200: title 1's terrain texture arrays and // layer pools, 170 MB-5.8 GB each, of which the game fills a little) get memory per block when the // GPU first writes there, not in full. nx2:gpu installs the handler and the span its regions cover; -// the GPU memory manager calls Nx2BackSparseWrite before each write. -using Nx2SparseBackingHandler = std::function<void(u64 gpu_address, u64 size)>; +// the GPU memory manager calls Nx2BackSparseWrite before each write, with the bytes it writes +// (blocks nobody wrote that only get zeros need no memory: they read zero already). +using Nx2SparseBackingHandler = std::function<void(u64 gpu_address, u64 size, const u8* data)>; inline std::recursive_mutex g_nx2_sparse_backing_mutex; inline Nx2SparseBackingHandler g_nx2_sparse_backing; inline std::atomic<u64> g_nx2_sparse_low{~u64{0}}; @@ -84,14 +123,14 @@ inline void Nx2AddSparseSpan(u64 gpu_address, u64 size) { } } -inline void Nx2BackSparseWrite(u64 gpu_address, u64 size) { +inline void Nx2BackSparseWrite(u64 gpu_address, u64 size, const void* data) { if (gpu_address + size <= g_nx2_sparse_low.load(std::memory_order_relaxed) || gpu_address >= g_nx2_sparse_high.load(std::memory_order_relaxed)) [[likely]] { return; } std::scoped_lock lk{g_nx2_sparse_backing_mutex}; if (g_nx2_sparse_backing) { - g_nx2_sparse_backing(gpu_address, size); + g_nx2_sparse_backing(gpu_address, size, static_cast<const u8*>(data)); } } @@ -136,6 +175,15 @@ inline void Nx2SparseBlockWritten(u64 region_address, u64 block) { } } +/// True unless the block of the region at region_address is known to hold nothing (no write from +/// the CPU or the GPU yet). +inline bool Nx2SparseBlockIsWritten(u64 region_address, u64 block) { + std::scoped_lock lk{g_nx2_sparse_state_mutex}; + const auto it = g_nx2_sparse_state.find(region_address); + return it == g_nx2_sparse_state.end() || block >= it->second.written.size() || + it->second.written[block]; +} + /// The game maps its own memory over [offset, +size) of the region at region_address (mapped), or /// unmaps it there. inline void Nx2SparseTilesMapped(u64 region_address, u64 offset, u64 size, bool mapped) { diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index b527c0e2a9..8486d44ac9 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -6,6 +6,7 @@ #pragma once +#include <array> #include <functional> #include <optional> #include <span> @@ -162,6 +163,16 @@ public: return false; } + /// Stand-in for a DLSS evaluation: temporal upscale of src into dst along motion_vectors + /// (jitter in src pixels; motion_scale turns a motion vector into a UV offset to the previous + /// frame). False when unsupported; the caller then falls back to AccelerateSurfaceCopy. + [[nodiscard]] virtual bool AccelerateTemporalUpscale( + const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, std::array<f32, 2> jitter, + std::array<f32, 2> motion_scale) { + return false; + } + [[nodiscard]] virtual Tegra::Engines::AccelerateDMAInterface& AccessAccelerateDMA() = 0; virtual void AccelerateInlineToMemory(GPUVAddr address, size_t copy_size, diff --git a/src/video_core/renderer_opengl/gl_compute_pipeline.cpp b/src/video_core/renderer_opengl/gl_compute_pipeline.cpp index f0a7baf9aa..eb134e86b5 100644 --- a/src/video_core/renderer_opengl/gl_compute_pipeline.cpp +++ b/src/video_core/renderer_opengl/gl_compute_pipeline.cpp @@ -12,6 +12,7 @@ #include "video_core/renderer_opengl/gl_compute_pipeline.h" #include "video_core/renderer_opengl/gl_shader_manager.h" #include "video_core/renderer_opengl/gl_shader_util.h" +#include "video_core/shader_handle_program.h" namespace OpenGL { @@ -105,6 +106,20 @@ void ComputePipeline::Configure() { const auto& cbufs{qmd.const_buffer_config}; const bool via_header_index{qmd.linked_tsc != 0}; const auto read_handle{[&](const auto& desc, u32 index) { + if constexpr (std::is_same_v<decltype(desc), const Shader::TextureDescriptor&>) { + if (desc.handle_program != ~0U) { + // Read by the shader from a storage buffer: computed for this dispatch. + return TexturePair(VideoCommon::EvaluateTextureHandle( + info, desc.handle_program, *gpu_memory, + [&](u32 cbuf) -> GPUVAddr { + return cbuf < cbufs.size() && + ((qmd.const_buffer_enable_mask >> cbuf) & 1) != 0 + ? cbufs[cbuf].Address() + : GPUVAddr{}; + }), + via_header_index); + } + } ASSERT(((qmd.const_buffer_enable_mask >> desc.cbuf_index) & 1) != 0); const u32 index_offset{index << desc.size_shift}; const u32 offset{desc.cbuf_offset + index_offset}; diff --git a/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp b/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp index ecd6544b10..81950279d5 100644 --- a/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp +++ b/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp @@ -17,6 +17,7 @@ #include "video_core/renderer_opengl/gl_shader_manager.h" #include "video_core/renderer_opengl/gl_shader_util.h" #include "video_core/renderer_opengl/gl_state_tracker.h" +#include "video_core/shader_handle_program.h" #include "video_core/shader_notify.h" #include "video_core/texture_cache/texture_cache.h" @@ -308,6 +309,19 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { } const auto& cbufs{maxwell3d->state.shader_stages[stage].const_buffers}; const auto read_handle{[&](const auto& desc, u32 index) { + if constexpr (std::is_same_v<decltype(desc), const Shader::TextureDescriptor&>) { + if (desc.handle_program != ~0U) { + // Read by the shader from a storage buffer: computed for this draw. + return TexturePair(VideoCommon::EvaluateTextureHandle( + info, desc.handle_program, *gpu_memory, + [&](u32 cbuf) -> GPUVAddr { + return cbuf < cbufs.size() && cbufs[cbuf].enabled + ? cbufs[cbuf].address + : GPUVAddr{}; + }), + via_header_index); + } + } ASSERT(cbufs[desc.cbuf_index].enabled); const u32 index_offset{index << desc.size_shift}; const u32 offset{desc.cbuf_offset + index_offset}; diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index 277cf222fa..ac750ab747 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -578,7 +578,8 @@ std::unique_ptr<ComputePipeline> ShaderCache::CreateComputePipeline( const ComputePipelineKey& key, const VideoCommon::ShaderInfo* shader) { const GPUVAddr program_base{kepler_compute->regs.code_loc.Address()}; const auto& qmd{kepler_compute->launch_description}; - ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start}; + ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start, + shader->lowered_native}; env.SetCachedSize(shader->size_bytes); main_pools.ReleaseContents(); diff --git a/src/video_core/renderer_opengl/present/layer.cpp b/src/video_core/renderer_opengl/present/layer.cpp index 9e4a0f854a..2656892f4a 100644 --- a/src/video_core/renderer_opengl/present/layer.cpp +++ b/src/video_core/renderer_opengl/present/layer.cpp @@ -183,6 +183,11 @@ void Layer::ConfigureFramebufferTexture(const Tegra::FramebufferConfig& framebuf framebuffer_texture.gl_format = GL_RGBA; framebuffer_texture.gl_type = GL_UNSIGNED_INT_8_8_8_8_REV; break; + case Service::android::PixelFormat::Rgba1010102: + internal_format = GL_RGB10_A2; + framebuffer_texture.gl_format = GL_RGBA; + framebuffer_texture.gl_type = GL_UNSIGNED_INT_2_10_10_10_REV; + break; case Service::android::PixelFormat::Rgb565: internal_format = GL_RGB565; framebuffer_texture.gl_format = GL_RGB; diff --git a/src/video_core/renderer_vulkan/blit_image.cpp b/src/video_core/renderer_vulkan/blit_image.cpp index ddd52e9426..edd151b563 100644 --- a/src/video_core/renderer_vulkan/blit_image.cpp +++ b/src/video_core/renderer_vulkan/blit_image.cpp @@ -33,6 +33,7 @@ #include "video_core/host_shaders/convert_non_msaa_to_msaa_depth_stencil_frag_spv.h" #include "video_core/host_shaders/convert_s8d24_to_abgr8_frag_spv.h" #include "video_core/host_shaders/full_screen_triangle_vert_spv.h" +#include "video_core/host_shaders/temporal_upscale_frag_spv.h" #include "video_core/host_shaders/vulkan_blit_depth_stencil_frag_spv.h" #include "video_core/host_shaders/vulkan_color_clear_frag_spv.h" #include "video_core/host_shaders/vulkan_color_clear_vert_spv.h" @@ -91,6 +92,15 @@ struct PushConstants { std::array<float, 2> tex_offset; }; +// The vertex stage's PushConstants first, then those of temporal_upscale.frag. +struct TemporalUpscalePushConstants { + PushConstants vertex; + std::array<float, 2> jitter; + std::array<float, 2> motion_scale; + float current_weight; + float padding; +}; + struct MSAACopyPushConstants { std::array<s32, 2> dst_offset; std::array<s32, 2> src_offset; @@ -109,6 +119,11 @@ constexpr std::array TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS{ TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<0>, TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<1>, }; +constexpr std::array THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS{ + TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<0>, + TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<1>, + TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<2>, +}; constexpr VkDescriptorSetLayoutCreateInfo ONE_TEXTURE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO{ .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .pNext = nullptr, @@ -133,6 +148,13 @@ constexpr VkDescriptorSetLayoutCreateInfo TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_CRE .bindingCount = static_cast<u32>(TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.size()), .pBindings = TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.data(), }; +constexpr VkDescriptorSetLayoutCreateInfo THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO{ + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .bindingCount = static_cast<u32>(THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.size()), + .pBindings = THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.data(), +}; template <VkShaderStageFlags stageFlags, size_t size> inline constexpr VkPushConstantRange PUSH_CONSTANT_RANGE{ .stageFlags = stageFlags, @@ -357,6 +379,32 @@ void UpdateOneTextureDescriptorSet(const Device& device, VkDescriptorSet descrip device.GetLogical().UpdateDescriptorSets(write_descriptor_set, nullptr); } +void UpdateThreeTexturesDescriptorSet(const Device& device, VkDescriptorSet descriptor_set, + VkSampler sampler, std::array<VkImageView, 3> image_views) { + std::array<VkDescriptorImageInfo, 3> image_infos{}; + std::array<VkWriteDescriptorSet, 3> write_descriptor_sets{}; + for (u32 binding = 0; binding < 3; ++binding) { + image_infos[binding] = VkDescriptorImageInfo{ + .sampler = sampler, + .imageView = image_views[binding], + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + }; + write_descriptor_sets[binding] = VkWriteDescriptorSet{ + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .pNext = nullptr, + .dstSet = descriptor_set, + .dstBinding = binding, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .pImageInfo = &image_infos[binding], + .pBufferInfo = nullptr, + .pTexelBufferView = nullptr, + }; + } + device.GetLogical().UpdateDescriptorSets(write_descriptor_sets, nullptr); +} + void UpdateTwoTexturesDescriptorSet(const Device& device, VkDescriptorSet descriptor_set, VkSampler sampler, VkImageView image_view_0, VkImageView image_view_1) { @@ -590,10 +638,14 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, ONE_TEXTURE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)), two_textures_set_layout(device.GetLogical().CreateDescriptorSetLayout( TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)), + three_textures_set_layout(device.GetLogical().CreateDescriptorSetLayout( + THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)), one_texture_descriptor_allocator{ descriptor_pool.Allocator(device_, scheduler_, *one_texture_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<1>)}, two_textures_descriptor_allocator{ descriptor_pool.Allocator(device_, scheduler_, *two_textures_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<2>)}, + three_textures_descriptor_allocator{ + descriptor_pool.Allocator(device_, scheduler_, *three_textures_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<3>)}, one_texture_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( one_texture_set_layout.address(), PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_VERTEX_BIT, sizeof(PushConstants)>))), @@ -601,6 +653,11 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( two_textures_set_layout.address(), PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_VERTEX_BIT, sizeof(PushConstants)>))), + temporal_upscale_pipeline_layout( + device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( + three_textures_set_layout.address(), + PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, + sizeof(TemporalUpscalePushConstants)>))), clear_color_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( nullptr, PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_FRAGMENT_BIT, sizeof(float) * 4>))), msaa_copy_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo( @@ -612,6 +669,7 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_FRAGMENT_BIT, sizeof(MSAACopyPushConstants)>))), full_screen_vert(BuildShader(device, FULL_SCREEN_TRIANGLE_VERT_SPV)), blit_color_to_color_frag(BuildShader(device, BLIT_COLOR_FLOAT_FRAG_SPV)), + temporal_upscale_frag(BuildShader(device, TEMPORAL_UPSCALE_FRAG_SPV)), blit_color_msaa_frag(BuildShader(device, BLIT_COLOR_MSAA_FRAG_SPV)), blit_depth_stencil_frag(device.IsExtShaderStencilExportSupported() ? BuildShader(device, VULKAN_BLIT_DEPTH_STENCIL_FRAG_SPV) @@ -702,6 +760,42 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, VkImageView }); } +void BlitImageHelper::TemporalUpscale(const Framebuffer* dst_framebuffer, + const ImageView& input_color, const ImageView& motion_vectors, + VkImageView history, const Region2D& dst_region, + std::array<f32, 2> jitter, std::array<f32, 2> motion_scale, + f32 current_weight) { + const VkPipelineLayout layout = *temporal_upscale_pipeline_layout; + const VkPipeline pipeline = FindOrEmplaceTemporalUpscalePipeline(dst_framebuffer->RenderPass()); + const VkSampler sampler = *linear_sampler; + const std::array<VkImageView, 3> views{input_color.Handle(Shader::TextureType::Color2D), + motion_vectors.Handle(Shader::TextureType::Color2D), + history}; + const TemporalUpscalePushConstants push_constants{ + .vertex = {.tex_scale = {1.0f, 1.0f}, .tex_offset = {0.0f, 0.0f}}, + .jitter = jitter, + .motion_scale = motion_scale, + .current_weight = current_weight, + .padding = 0.0f, + }; + RecordShaderReadBarrier(scheduler, input_color); + RecordShaderReadBarrier(scheduler, motion_vectors); + scheduler.RequestRenderpass(dst_framebuffer); + scheduler.Record([this, dst_region, pipeline, layout, sampler, views, + push_constants](vk::CommandBuffer cmdbuf) { + const VkDescriptorSet descriptor_set = three_textures_descriptor_allocator.Commit(); + UpdateThreeTexturesDescriptorSet(device, descriptor_set, sampler, views); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, descriptor_set, + nullptr); + BindBlitState(cmdbuf, dst_region); + cmdbuf.PushConstants(layout, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, + push_constants); + cmdbuf.Draw(3, 1, 0, 0); + }); + scheduler.InvalidateState(); +} + void BlitImageHelper::BlitImpl(const Framebuffer* dst_framebuffer, const ImageView& src_image_view, const Region2D& dst_region, const Region2D& src_region, VkPipeline pipeline, VkSampler sampler, @@ -1285,6 +1379,59 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe return *blit_color_pipelines.back(); } +VkPipeline BlitImageHelper::FindOrEmplaceTemporalUpscalePipeline(VkRenderPass renderpass) { + const auto it = std::ranges::find(temporal_upscale_keys, renderpass); + if (it != temporal_upscale_keys.end()) { + return *temporal_upscale_pipelines[std::distance(temporal_upscale_keys.begin(), it)]; + } + temporal_upscale_keys.push_back(renderpass); + const std::array stages = MakeStages(*full_screen_vert, *temporal_upscale_frag); + const VkPipelineColorBlendAttachmentState blend_attachment{ + .blendEnable = VK_FALSE, + .srcColorBlendFactor = VK_BLEND_FACTOR_ZERO, + .dstColorBlendFactor = VK_BLEND_FACTOR_ZERO, + .colorBlendOp = VK_BLEND_OP_ADD, + .srcAlphaBlendFactor = VK_BLEND_FACTOR_ZERO, + .dstAlphaBlendFactor = VK_BLEND_FACTOR_ZERO, + .alphaBlendOp = VK_BLEND_OP_ADD, + .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | + VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, + }; + const VkPipelineColorBlendStateCreateInfo color_blend_create_info{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .logicOpEnable = VK_FALSE, + .logicOp = VK_LOGIC_OP_CLEAR, + .attachmentCount = 1, + .pAttachments = &blend_attachment, + .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}, + }; + const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); + temporal_upscale_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ + .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .stageCount = static_cast<u32>(stages.size()), + .pStages = stages.data(), + .pVertexInputState = &PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO, + .pInputAssemblyState = &input_assembly_ci, + .pTessellationState = nullptr, + .pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO, + .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO, + .pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, + .pDepthStencilState = nullptr, + .pColorBlendState = &color_blend_create_info, + .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, + .layout = *temporal_upscale_pipeline_layout, + .renderPass = renderpass, + .subpass = 0, + .basePipelineHandle = VK_NULL_HANDLE, + .basePipelineIndex = 0, + }, device.StaticPipelineCache())); + return *temporal_upscale_pipelines.back(); +} + VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key) { const auto it = std::ranges::find(blit_depth_stencil_keys, key); if (it != blit_depth_stencil_keys.end()) { diff --git a/src/video_core/renderer_vulkan/blit_image.h b/src/video_core/renderer_vulkan/blit_image.h index 62e6743a23..303beeea5c 100644 --- a/src/video_core/renderer_vulkan/blit_image.h +++ b/src/video_core/renderer_vulkan/blit_image.h @@ -82,6 +82,13 @@ public: VkImage src_image, VkSampler src_sampler, const Region2D& dst_region, const Region2D& src_region, const Extent3D& src_size); + /// Temporal upscale (temporal_upscale.frag) into dst_framebuffer: the jittered input colour, + /// accumulated with the history image reprojected along the motion vectors. + void TemporalUpscale(const Framebuffer* dst_framebuffer, const ImageView& input_color, + const ImageView& motion_vectors, VkImageView history, + const Region2D& dst_region, std::array<f32, 2> jitter, + std::array<f32, 2> motion_scale, f32 current_weight); + void BlitColorMSAA(const Framebuffer* dst_framebuffer, const ImageView& src_image_view, const Region2D& dst_region, const Region2D& src_region); @@ -170,6 +177,8 @@ private: [[nodiscard]] VkPipeline FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key); + [[nodiscard]] VkPipeline FindOrEmplaceTemporalUpscalePipeline(VkRenderPass renderpass); + [[nodiscard]] VkPipeline FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key); [[nodiscard]] VkPipeline FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key); @@ -207,15 +216,19 @@ private: vk::DescriptorSetLayout one_texture_set_layout; vk::DescriptorSetLayout two_textures_set_layout; + vk::DescriptorSetLayout three_textures_set_layout; DescriptorAllocator one_texture_descriptor_allocator; DescriptorAllocator two_textures_descriptor_allocator; + DescriptorAllocator three_textures_descriptor_allocator; vk::PipelineLayout one_texture_pipeline_layout; vk::PipelineLayout two_textures_pipeline_layout; + vk::PipelineLayout temporal_upscale_pipeline_layout; vk::PipelineLayout clear_color_pipeline_layout; vk::PipelineLayout msaa_copy_pipeline_layout; vk::PipelineLayout msaa_copy_depth_stencil_pipeline_layout; vk::ShaderModule full_screen_vert; vk::ShaderModule blit_color_to_color_frag; + vk::ShaderModule temporal_upscale_frag; vk::ShaderModule blit_color_msaa_frag; vk::ShaderModule blit_depth_stencil_frag; vk::ShaderModule blit_depth_frag; @@ -246,6 +259,8 @@ private: std::vector<BlitImagePipelineKey> blit_color_keys; std::vector<vk::Pipeline> blit_color_pipelines; + std::vector<VkRenderPass> temporal_upscale_keys; + std::vector<vk::Pipeline> temporal_upscale_pipelines; std::vector<BlitImagePipelineKey> blit_depth_stencil_keys; std::vector<vk::Pipeline> blit_depth_stencil_pipelines; std::vector<BlitImagePipelineKey> clear_color_keys; diff --git a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp index 9dac98c830..d1f78c0034 100644 --- a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp +++ b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp @@ -14,6 +14,7 @@ #include "common/settings.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/renderer_vulkan/fixed_pipeline_state.h" +#include "video_core/renderer_vulkan/vertex_repack.h" #include "video_core/renderer_vulkan/vk_state_tracker.h" namespace Vulkan { @@ -268,6 +269,18 @@ void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFe if (xfb_enabled) { RefreshXfbState(xfb_state, regs); } + if (!dynamic_vertex_input && ExperimentalStaticVertexRepackEnabled()) { + const u32 repack_mask = VertexRepackMask(regs); + // Refresh from guest values even when only the stride changed: offsets must not accumulate. + for (size_t index = 0; index < attributes.size(); ++index) { + const auto& input = regs.vertex_attrib_format[index]; + attributes[index].offset.Assign(RepackedVertexOffset(repack_mask, input.buffer, input.offset)); + } + for (u32 binding = 0; binding < vertex_strides.size(); ++binding) { + vertex_strides[binding] = static_cast<u16>(RepackedVertexStride( + repack_mask, binding, regs.vertex_streams[binding].stride)); + } + } } void FixedPipelineState::BlendingAttachment::Refresh(const Maxwell& regs, size_t index) { diff --git a/src/video_core/renderer_vulkan/maxwell_to_vk.cpp b/src/video_core/renderer_vulkan/maxwell_to_vk.cpp index c8e43d0650..e03e5786fe 100644 --- a/src/video_core/renderer_vulkan/maxwell_to_vk.cpp +++ b/src/video_core/renderer_vulkan/maxwell_to_vk.cpp @@ -333,6 +333,10 @@ VkShaderStageFlagBits ShaderStage(Shader::Stage stage) { return VK_SHADER_STAGE_FRAGMENT_BIT; case Shader::Stage::Compute: return VK_SHADER_STAGE_COMPUTE_BIT; + case Shader::Stage::Task: + return VK_SHADER_STAGE_TASK_BIT_EXT; + case Shader::Stage::Mesh: + return VK_SHADER_STAGE_MESH_BIT_EXT; } UNIMPLEMENTED_MSG("Unimplemented shader stage={}", stage); return {}; diff --git a/src/video_core/renderer_vulkan/present/layer.cpp b/src/video_core/renderer_vulkan/present/layer.cpp index fa24c02f6d..5652090bd7 100644 --- a/src/video_core/renderer_vulkan/present/layer.cpp +++ b/src/video_core/renderer_vulkan/present/layer.cpp @@ -52,6 +52,8 @@ VkFormat GetFormat(const Tegra::FramebufferConfig& framebuffer) { return VK_FORMAT_R5G6B5_UNORM_PACK16; case Service::android::PixelFormat::Bgra8888: return VK_FORMAT_B8G8R8A8_UNORM; + case Service::android::PixelFormat::Rgba1010102: + return VK_FORMAT_A2B10G10R10_UNORM_PACK32; default: UNIMPLEMENTED_MSG("Unknown framebuffer pixel format: {}", static_cast<u32>(framebuffer.pixel_format)); diff --git a/src/video_core/renderer_vulkan/vertex_repack.h b/src/video_core/renderer_vulkan/vertex_repack.h new file mode 100644 index 0000000000..b0849c25c7 --- /dev/null +++ b/src/video_core/renderer_vulkan/vertex_repack.h @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <cstdlib> +#include <cstring> +#include <limits> +#include <optional> +#include "video_core/engines/maxwell_3d.h" + +namespace Vulkan { + +constexpr bool VertexRepackInputSupported(bool dynamic_input, bool experimental_static) { + return dynamic_input || experimental_static; +} + +inline bool ExperimentalStaticVertexRepackEnabled() { + static const bool enabled = [] { + const char* value = std::getenv("EGLANTINE_EXPERIMENTAL_STATIC_VERTEX_REPACK"); + return value && std::strcmp(value, "1") == 0; + }(); + return enabled; +} + +struct VertexRepackParameters { + u32 vertices; + u32 output_size; + u32 source_base; + u32 source_bias; + u32 source_range; + u32 dispatch_groups; +}; + +inline std::optional<VertexRepackParameters> MakeVertexRepackParameters( + u32 offset, u32 size, u64 alignment, u64 max_storage_range, u32 max_dispatch_groups) { + if (size == 0 || alignment == 0) { + return std::nullopt; + } + const u64 vertices = (u64{size} + 68) / 69; + const u64 output_size = vertices * 72; + const u64 base = u64{offset} / alignment * alignment; + const u64 bias = offset - base; + const u64 range = (bias + size + 3) / 4 * 4; + const u64 groups = (vertices * 18 + 127) / 128; + if (output_size > std::numeric_limits<u32>::max() || + range > std::numeric_limits<u32>::max() || output_size > max_storage_range || + range > max_storage_range || groups > max_dispatch_groups) { + return std::nullopt; + } + return VertexRepackParameters{static_cast<u32>(vertices), static_cast<u32>(output_size), + static_cast<u32>(base), static_cast<u32>(bias), + static_cast<u32>(range), static_cast<u32>(groups)}; +} + +inline bool VertexRepackEnabled() { + static const bool enabled = [] { + const char* value = std::getenv("EGLANTINE_TEST_VERTEX_REPACK"); + return !value || std::strcmp(value, "0") != 0; + }(); + return enabled; +} + +inline u32 VertexRepackMask(const Tegra::Engines::Maxwell3D::Regs& regs) { + using Attribute = Tegra::Engines::Maxwell3D::Regs::VertexAttribute; + if (!VertexRepackEnabled()) { + return 0; + } + u32 candidates = 0; + u32 excluded = 0; + for (const auto& attribute : regs.vertex_attrib_format) { + const u32 binding = attribute.buffer; + if (attribute.constant || regs.vertex_streams[binding].stride != 69) { + continue; + } + if (attribute.size == Attribute::Size::Invalid) { + excluded |= 1U << binding; + continue; + } + const u32 offset = attribute.offset; + const u32 size = attribute.SizeInBytes(); + if (offset == 65 && size == 4) { + candidates |= 1U << binding; + } + if (offset + size > 69 || (offset < 65 && offset + size > 65)) { + excluded |= 1U << binding; + } + } + return candidates & ~excluded; +} + +inline u32 RepackedVertexOffset(u32 mask, u32 binding, u32 offset) { + return offset + ((mask & (1U << binding)) && offset >= 65 ? 3U : 0U); +} + +inline u32 RepackedVertexStride(u32 mask, u32 binding, u32 stride) { + return mask & (1U << binding) ? 72U : stride; +} + +} // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 493de2cc20..fe0ab3a28e 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -7,11 +7,13 @@ #include <algorithm> #include <array> #include <cstring> +#include <cstdlib> #include <span> #include <vector> #include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/renderer_vulkan/vk_buffer_cache.h" +#include "video_core/renderer_vulkan/vertex_repack.h" #include "video_core/renderer_vulkan/maxwell_to_vk.h" #include "video_core/renderer_vulkan/vk_scheduler.h" @@ -359,6 +361,14 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m compute_pass_descriptor_queue), multi_range_buffers(device_) { const VkDriverIdKHR driver_id = device.GetDriverID(); + const bool enable_vertex_repack = VertexRepackEnabled(); + LOG_INFO(Render_Vulkan, "69-byte vertex layout repacking {} ({})", + enable_vertex_repack ? "enabled" : "disabled", + "portable Vulkan layout fallback"); + if (enable_vertex_repack) { + vertex_repack_pass = std::make_unique<VertexRepackPass>( + device, scheduler, descriptor_pool, staging_pool, compute_pass_descriptor_queue); + } limit_dynamic_storage_buffers = driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY || driver_id == VK_DRIVER_ID_ARM_PROPRIETARY; if (limit_dynamic_storage_buffers) { @@ -374,6 +384,19 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m scheduler_, staging_pool_); } +u32 BufferCacheRuntime::SetVertexRepackLayout(const Tegra::Engines::Maxwell3D::Regs& regs, + bool dynamic_input) { + const u32 old_mask = vertex_repack_mask; + const bool input_supported = VertexRepackInputSupported( + dynamic_input, ExperimentalStaticVertexRepackEnabled()); + vertex_repack_mask = vertex_repack_pass && input_supported ? VertexRepackMask(regs) : 0; + if (vertex_repack_mask != old_mask) { + LOG_DEBUG(Render_Vulkan, "Vertex repack selection: mask={:#x}, dynamic_input={}", + vertex_repack_mask, dynamic_input); + } + return old_mask | vertex_repack_mask; +} + StagingBufferRef BufferCacheRuntime::UploadStagingBuffer(size_t size) { return staging_pool.Request(size, MemoryUsage::Upload); } @@ -629,6 +652,16 @@ void BufferCacheRuntime::BindVertexBuffer(u32 index, VkBuffer buffer, u32 offset if (index >= device.GetMaxVertexInputBindings()) { return; } + if (buffer != VK_NULL_HANDLE && size != 0 && IsVertexRepacked(index)) { + const auto repacked = vertex_repack_pass->Assemble(buffer, offset, size); + if (!repacked) { + return; + } + buffer = repacked->first; + offset = static_cast<u32>(repacked->second); + size = static_cast<u32>((u64{size} + 68) / 69 * 72); + stride = 72; + } if (device.IsExtExtendedDynamicStateSupported()) { scheduler.Record([index, buffer, offset, size, stride](vk::CommandBuffer cmdbuf) { const VkDeviceSize vk_offset = buffer != VK_NULL_HANDLE ? offset : 0; @@ -661,6 +694,24 @@ void BufferCacheRuntime::BindVertexBuffers(VideoCommon::HostBindings<Buffer>& bi } } buffer_handles[i] = handle; + const u32 binding = bindings.min_index + i; + if (handle != VK_NULL_HANDLE && IsVertexRepacked(binding) && bindings.strides[i] == 69 && + bindings.sizes[i] > 0) { + const auto repacked = vertex_repack_pass->Assemble( + handle, static_cast<u32>(bindings.offsets[i]), static_cast<u32>(bindings.sizes[i])); + if (!repacked) { + continue; + } + buffer_handles[i] = repacked->first; + bindings.offsets[i] = repacked->second; + bindings.sizes[i] = (bindings.sizes[i] + 68) / 69 * 72; + bindings.strides[i] = 72; + static bool reported = false; + if (!reported) { + LOG_WARNING(Render_Vulkan, "Experimental GPU vertex repacking active: 69 -> 72 bytes"); + reported = true; + } + } } const u32 device_max = device.GetMaxVertexInputBindings(); const u32 min_binding = (std::min)(bindings.min_index, device_max); diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index fff1f8164d..ff854445b5 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -110,6 +110,12 @@ public: void TickFrame(Common::SlotVector<Buffer>& slot_buffers) noexcept; + u32 SetVertexRepackLayout(const Tegra::Engines::Maxwell3D::Regs& regs, bool dynamic_input); + + bool IsVertexRepacked(u32 binding) const { + return (vertex_repack_mask & (1U << binding)) != 0; + } + u64 CurrentTick(); bool IsFree(u64 tick); @@ -259,6 +265,8 @@ private: vk::Buffer null_buffer; std::unique_ptr<Uint8Pass> uint8_pass; + std::unique_ptr<VertexRepackPass> vertex_repack_pass; + u32 vertex_repack_mask{}; QuadIndexedPass quad_index_pass; MultiRangeBufferCache multi_range_buffers; diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp index 8cd8abb599..cb8dcde42f 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp @@ -26,8 +26,10 @@ #include "video_core/host_shaders/resolve_conditional_render_comp_spv.h" #include "video_core/host_shaders/vulkan_quad_indexed_comp_spv.h" #include "video_core/host_shaders/vulkan_uint8_comp_spv.h" +#include "video_core/host_shaders/vulkan_vertex_repack_comp_spv.h" #include "video_core/host_shaders/block_linear_unswizzle_3d_bcn_comp_spv.h" #include "video_core/renderer_vulkan/vk_compute_pass.h" +#include "video_core/renderer_vulkan/vertex_repack.h" #include "video_core/surface.h" #include "video_core/renderer_vulkan/vk_descriptor_pool.h" #include "video_core/renderer_vulkan/vk_scheduler.h" @@ -324,6 +326,62 @@ ComputePass::ComputePass(const Device& device_, Scheduler& scheduler, Descriptor ComputePass::~ComputePass() = default; +VertexRepackPass::VertexRepackPass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool, StagingBufferPool& staging_pool_, + ComputePassDescriptorQueue& descriptor_queue_) + : ComputePass(device_, scheduler_, descriptor_pool, INPUT_OUTPUT_DESCRIPTOR_SET_BINDINGS, + INPUT_OUTPUT_DESCRIPTOR_UPDATE_TEMPLATE, INPUT_OUTPUT_BANK_INFO, + COMPUTE_PUSH_CONSTANT_RANGE<12>, VULKAN_VERTEX_REPACK_COMP_SPV), + scheduler{scheduler_}, staging_pool{staging_pool_}, descriptor_queue{descriptor_queue_} {} + +std::optional<std::pair<VkBuffer, VkDeviceSize>> VertexRepackPass::Assemble(VkBuffer source, + u32 offset, u32 size) { + const auto parameters = MakeVertexRepackParameters( + offset, size, device.GetStorageBufferAlignment(), device.GetMaxStorageBufferRange(), + device.GetMaxComputeWorkGroupCount()[0]); + if (!parameters) { + LOG_ERROR(Render_Vulkan, "Vertex repack exceeds device limits: offset={}, size={}", offset, size); + return std::nullopt; + } + const u32 num_vertices = parameters->vertices; + const u32 output_size = parameters->output_size; + const auto dest = staging_pool.Request(output_size, MemoryUsage::DeviceLocal); + const u32 base_offset = parameters->source_base; + const u32 bias = parameters->source_bias; + descriptor_queue.Acquire(scheduler, 2); + descriptor_queue.AddBuffer(source, base_offset, parameters->source_range); + descriptor_queue.AddBuffer(dest.buffer, dest.offset, output_size); + const void* data = descriptor_queue.UpdateData(); + scheduler.RequestOutsideRenderPassOperationContext(); + LOG_DEBUG(Render_Vulkan, "Vertex repack queued: source_size={}, output_size={}, groups={}", + size, output_size, parameters->dispatch_groups); + scheduler.Record([this, data, bias, size, num_vertices, + groups = parameters->dispatch_groups](vk::CommandBuffer cmdbuf) { + const VkMemoryBarrier read_barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, read_barrier); + const auto set = descriptor_allocator.Commit(); + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, data); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + const std::array<u32, 3> constants{bias, size, num_vertices * 18}; + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(constants), constants.data()); + cmdbuf.Dispatch(groups, 1, 1); + const VkMemoryBarrier write_barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, + VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, 0, write_barrier); + }); + return std::pair<VkBuffer, VkDeviceSize>{dest.buffer, dest.offset}; +} + Uint8Pass::Uint8Pass(const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool, StagingBufferPool& staging_buffer_pool_, ComputePassDescriptorQueue& compute_pass_descriptor_queue_) @@ -743,11 +801,11 @@ void ASTCDecoderPass::AssembleInto(const VideoCommon::ImageInfo& info, u32 skip, cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, image_barrier); }); - // Each decode ends with the GPU thread waiting for the host GPU to go idle. Measured here - // (count and time per 5 s); EDEN_NX2_ASTC_FLUSH=1 submits without waiting instead. + // Each decode ends with a submit to the host GPU. Measured here + // (count and time per 5 s); EDEN_NX2_ASTC_FLUSH=0 waits for the host GPU to go idle instead. static const bool flush_only = [] { const char* value = std::getenv("EDEN_NX2_ASTC_FLUSH"); - return value != nullptr && value[0] == '1'; + return value == nullptr || value[0] != '0'; }(); if (!target.wait) { return; diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.h b/src/video_core/renderer_vulkan/vk_compute_pass.h index 9f2e8e9f1c..27fffbbbb7 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.h +++ b/src/video_core/renderer_vulkan/vk_compute_pass.h @@ -56,6 +56,20 @@ private: vk::ShaderModule module; }; +class VertexRepackPass final : public ComputePass { +public: + VertexRepackPass(const Device& device, Scheduler& scheduler, DescriptorPool& descriptor_pool, + StagingBufferPool& staging_pool, ComputePassDescriptorQueue& descriptor_queue); + /// Returns nothing when the range exceeds the device limits; the binding is then left as is. + std::optional<std::pair<VkBuffer, VkDeviceSize>> Assemble(VkBuffer source, u32 offset, + u32 size); + +private: + Scheduler& scheduler; + StagingBufferPool& staging_pool; + ComputePassDescriptorQueue& descriptor_queue; +}; + class Uint8Pass final : public ComputePass { public: explicit Uint8Pass(const Device& device_, Scheduler& scheduler_, diff --git a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp index 29356cfcec..1cfe391ba6 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp @@ -18,6 +18,7 @@ #include "video_core/renderer_vulkan/vk_pipeline_cache.h" #include "video_core/renderer_vulkan/vk_scheduler.h" #include "video_core/renderer_vulkan/vk_update_descriptor.h" +#include "video_core/shader_handle_program.h" #include "video_core/shader_notify.h" #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -134,6 +135,9 @@ ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk if (pipeline_statistics) { pipeline_statistics->Collect(device, *pipeline); } + // Nothing reads the module once the pipeline exists; the driver keeps its SPIR-V while it + // lives. + spv_module.reset(); std::scoped_lock lock{build_mutex}; is_built = true; build_condvar.notify_one(); @@ -172,6 +176,20 @@ bool ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute, const auto& cbufs{qmd.const_buffer_config}; const bool via_header_index{qmd.linked_tsc != 0}; const auto read_handle{[&](const auto& desc, u32 index) { + if constexpr (std::is_same_v<decltype(desc), const Shader::TextureDescriptor&>) { + if (desc.handle_program != ~0U) { + // Read by the shader from a storage buffer: computed for this dispatch. + return TexturePair(VideoCommon::EvaluateTextureHandle( + info, desc.handle_program, gpu_memory, + [&](u32 cbuf) -> GPUVAddr { + return cbuf < cbufs.size() && + ((qmd.const_buffer_enable_mask >> cbuf) & 1) != 0 + ? cbufs[cbuf].Address() + : GPUVAddr{}; + }), + via_header_index); + } + } ASSERT(((qmd.const_buffer_enable_mask >> desc.cbuf_index) & 1) != 0); const u32 index_offset{index << desc.size_shift}; const u32 offset{desc.cbuf_offset + index_offset}; diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp index 94bace432c..fa273f6ef9 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp @@ -24,6 +24,7 @@ #include "video_core/renderer_vulkan/vk_scheduler.h" #include "video_core/renderer_vulkan/vk_texture_cache.h" #include "video_core/renderer_vulkan/vk_update_descriptor.h" +#include "video_core/shader_handle_program.h" #include "video_core/shader_notify.h" #include "video_core/texture_cache/texture_cache.h" #include "video_core/vulkan_common/vulkan_device.h" @@ -54,6 +55,17 @@ using VideoCore::Surface::PixelFormatFromRenderTargetFormat; constexpr size_t NUM_STAGES = Maxwell::MaxShaderStage; +// The stage of a program: task and mesh programs sit in the vertex and tessellation slots. +VkShaderStageFlagBits StageFlag(const Shader::Info& info, VkShaderStageFlagBits slot_stage) { + if (info.is_task) { + return VK_SHADER_STAGE_TASK_BIT_EXT; + } + if (info.is_mesh) { + return VK_SHADER_STAGE_MESH_BIT_EXT; + } + return slot_stage; +} + DescriptorLayoutBuilder MakeBuilder(const Device& device, std::span<const Shader::Info> infos) { DescriptorLayoutBuilder builder{device}; for (size_t index = 0; index < infos.size(); ++index) { @@ -64,7 +76,7 @@ DescriptorLayoutBuilder MakeBuilder(const Device& device, std::span<const Shader VK_SHADER_STAGE_GEOMETRY_BIT, VK_SHADER_STAGE_FRAGMENT_BIT, }; - builder.Add(infos[index], stages.at(index)); + builder.Add(infos[index], StageFlag(infos[index], stages.at(index))); } return builder; } @@ -330,6 +342,11 @@ GraphicsPipeline::GraphicsPipeline( if (pipeline_statistics) { pipeline_statistics->Collect(device, *pipeline); } + // Nothing reads the shader modules once the pipeline exists, and the driver keeps a copy + // of each one's SPIR-V while it lives (title 1: ~1,500 pipelines, 2-5 modules each). + for (vk::ShaderModule& module : spv_modules) { + module.reset(); + } std::scoped_lock lock{build_mutex}; is_built = true; @@ -338,12 +355,13 @@ GraphicsPipeline::GraphicsPipeline( shader_notify->MarkShaderComplete(); } }}; + // Before the build: it frees the modules this looks at. + configure_func = ConfigureFunc(spv_modules, stage_infos); if (worker_thread) { worker_thread->QueueWork(std::move(func)); } else { func(); } - configure_func = ConfigureFunc(spv_modules, stage_infos); } void GraphicsPipeline::AddTransition(GraphicsPipeline* transition) { @@ -382,6 +400,19 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { } const auto& cbufs{maxwell3d->state.shader_stages[stage].const_buffers}; const auto read_handle{[&](const auto& desc, u32 index) { + if constexpr (std::is_same_v<decltype(desc), const Shader::TextureDescriptor&>) { + if (desc.handle_program != ~0U) { + // Read by the shader from a storage buffer: computed for this draw. + return TexturePair(VideoCommon::EvaluateTextureHandle( + info, desc.handle_program, *gpu_memory, + [&](u32 cbuf) -> GPUVAddr { + return cbuf < cbufs.size() && cbufs[cbuf].enabled + ? cbufs[cbuf].address + : GPUVAddr{}; + }), + via_header_index); + } + } ASSERT(cbufs[desc.cbuf_index].enabled); const u32 index_offset{index << desc.size_shift}; const u32 offset{desc.cbuf_offset + index_offset}; @@ -1073,6 +1104,19 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { } } + const bool is_mesh{IsMeshPipeline()}; + if (is_mesh) { + // Mesh pipelines have no vertex input or input assembly. + dynamic_states.erase(std::remove_if(dynamic_states.begin(), dynamic_states.end(), + [](VkDynamicState state) { + return state == VK_DYNAMIC_STATE_PRIMITIVE_TOPOLOGY_EXT || + state == VK_DYNAMIC_STATE_PRIMITIVE_RESTART_ENABLE_EXT || + state == VK_DYNAMIC_STATE_VERTEX_INPUT_EXT || + state == VK_DYNAMIC_STATE_VERTEX_INPUT_BINDING_STRIDE_EXT || + state == VK_DYNAMIC_STATE_PATCH_CONTROL_POINTS_EXT; + }), + dynamic_states.end()); + } const VkPipelineDynamicStateCreateInfo dynamic_state_ci{ .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO, .pNext = nullptr, @@ -1095,7 +1139,8 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .pNext = nullptr, .flags = 0, - .stage = MaxwellToVK::ShaderStage(Shader::StageFromIndex(stage)), + .stage = StageFlag(stage_infos[stage], + MaxwellToVK::ShaderStage(Shader::StageFromIndex(stage))), .module = *spv_modules[stage], .pName = "main", .pSpecializationInfo = nullptr, @@ -1115,9 +1160,9 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { .flags = flags, .stageCount = static_cast<u32>(shader_stages.size()), .pStages = shader_stages.data(), - .pVertexInputState = &vertex_input_ci, - .pInputAssemblyState = &input_assembly_ci, - .pTessellationState = &tessellation_ci, + .pVertexInputState = is_mesh ? nullptr : &vertex_input_ci, + .pInputAssemblyState = is_mesh ? nullptr : &input_assembly_ci, + .pTessellationState = is_mesh ? nullptr : &tessellation_ci, .pViewportState = &viewport_ci, .pRasterizationState = &rasterization_ci, .pMultisampleState = &multisample_ci, diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h index b1ac5a3fcc..6d6e2ab12d 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h @@ -98,6 +98,11 @@ public: bool UsesExtendedDynamicState() const noexcept { return key.state.extended_dynamic_state != 0; } + + /// Switch 2 task/mesh pipeline: drawn with vkCmdDrawMeshTasksEXT. + [[nodiscard]] bool IsMeshPipeline() const noexcept { + return stage_infos[0].is_task || stage_infos[0].is_mesh; + } GraphicsPipeline& operator=(GraphicsPipeline&&) noexcept = delete; GraphicsPipeline(GraphicsPipeline&&) noexcept = delete; @@ -119,6 +124,21 @@ public: : nullptr; } + /// The frame the pipeline cache last handed this pipeline out (evicts long unused ones). + void MarkUsed(u64 frame) noexcept { + last_use_frame = frame; + } + + [[nodiscard]] u64 LastUse() const noexcept { + return last_use_frame; + } + + /// Forgets the transitions, which may point to pipelines the cache evicted. + void ClearTransitions() noexcept { + transition_keys.clear(); + transitions.clear(); + } + [[nodiscard]] bool IsBuilt() const noexcept { return is_built.load(std::memory_order::relaxed); } @@ -159,6 +179,7 @@ private: std::vector<GraphicsPipelineCacheKey> transition_keys; std::vector<GraphicsPipeline*> transitions; + u64 last_use_frame{}; std::array<vk::ShaderModule, NUM_STAGES> spv_modules; diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index 05511c10e1..ff8e84b019 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -170,6 +170,14 @@ Shader::RuntimeInfo MakeRuntimeInfo(std::span<const Shader::IR::Program> program } else { info.previous_stage_stores.mask.set(); } + if (previous_program && previous_program->stage == Shader::Stage::Mesh) { + for (const u32 line : previous_program->mesh.primitive_lines) { + const auto attribute{static_cast<Shader::IR::Attribute>(line)}; + if (Shader::IR::IsGeneric(attribute)) { + info.per_primitive_generics.set(Shader::IR::GenericAttributeIndex(attribute)); + } + } + } const Shader::Stage stage{program.stage}; const bool has_geometry{key.unique_hashes[4] != 0 && !programs[4].is_geometry_passthrough}; const bool gl_ndc{key.state.ndc_minus_one_to_one != 0}; @@ -230,6 +238,9 @@ Shader::RuntimeInfo MakeRuntimeInfo(std::span<const Shader::IR::Program> program return Shader::TessSpacing::Equal; }(); break; + case Shader::Stage::Mesh: + info.convert_depth_mode = gl_ndc; + break; case Shader::Stage::Geometry: if (program.output_topology == Shader::OutputTopology::PointList) { info.fixed_state_point_size = point_size; @@ -345,6 +356,22 @@ bool GraphicsPipelineCacheKey::operator==(const GraphicsPipelineCacheKey& rhs) c return std::memcmp(&rhs, this, Size()) == 0; } +namespace { +// NVIDIA's and AMD's drivers keep a disk cache of built pipelines of their own. Eden's copy on top +// only kept every pipeline the title ever built in host memory (title 1: a 245 MB file, ~0.35 GB +// of RAM for the session) and copied it whole at each save, which stalled frames; pipelines built +// no slower without it. +bool DriverCachesPipelines(const Device& device) { + switch (device.GetDriverID()) { + case VK_DRIVER_ID_NVIDIA_PROPRIETARY: + case VK_DRIVER_ID_AMD_PROPRIETARY: + return true; + default: + return false; + } +} +} // Anonymous namespace + PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, @@ -358,7 +385,8 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, render_pass_cache{render_pass_cache_}, buffer_cache{buffer_cache_}, texture_cache{texture_cache_}, shader_notify{shader_notify_}, use_asynchronous_shaders{Settings::values.use_asynchronous_shaders.GetValue()}, - use_vulkan_pipeline_cache{Settings::values.use_vulkan_driver_pipeline_cache.GetValue()}, + use_vulkan_pipeline_cache{Settings::values.use_vulkan_driver_pipeline_cache.GetValue() && + !DriverCachesPipelines(device_)}, workers(device.HasBrokenParallelShaderCompiling() ? 1ULL : GetTotalPipelineWorkers(), "VkPipelineBuilder", {}, Common::ThreadPlacement::Background), serialization_thread(1, "VkPipelineSerialization", {}, @@ -378,7 +406,9 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, Shader::Stage::TessellationEval) | subgroup_stage_bit(VK_SHADER_STAGE_GEOMETRY_BIT, Shader::Stage::Geometry) | subgroup_stage_bit(VK_SHADER_STAGE_FRAGMENT_BIT, Shader::Stage::Fragment) | - subgroup_stage_bit(VK_SHADER_STAGE_COMPUTE_BIT, Shader::Stage::Compute)}; + subgroup_stage_bit(VK_SHADER_STAGE_COMPUTE_BIT, Shader::Stage::Compute) | + subgroup_stage_bit(VK_SHADER_STAGE_TASK_BIT_EXT, Shader::Stage::Task) | + subgroup_stage_bit(VK_SHADER_STAGE_MESH_BIT_EXT, Shader::Stage::Mesh)}; if (driver_id == VK_DRIVER_ID_MESA_TURNIP) { supported_subgroup_stages &= ~(1u << static_cast<u32>(Shader::Stage::Geometry)); } @@ -563,13 +593,96 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, device.SupportsTransformFeedbackProvokingVertexPreservation(); } +namespace { +// <title>/vulkan_used.bin: the graphics pipelines recent sessions used, as key hash + sessions +// since the last use. +constexpr u32 UsedPipelinesMagic = 0x31505545; // "EUP1" +// Pipelines the last two sessions used are built at boot. Each holds ~0.45 MB of driver memory: +// title 1 built 4,800 from eight sessions, 1.5 GB more than building them on first use (but that +// stalled frames 4x as often), while one session of its title screen uses ~1,500. +constexpr u8 UsedPipelinesMaxAge = 2; + +std::unordered_map<u64, u8> LoadUsedPipelines(const std::filesystem::path& path) { + std::unordered_map<u64, u8> used; + std::ifstream file{path, std::ios::binary}; + u32 magic{}, count{}; + if (!file.read(reinterpret_cast<char*>(&magic), sizeof(magic)) || magic != UsedPipelinesMagic || + !file.read(reinterpret_cast<char*>(&count), sizeof(count))) { + return used; + } + used.reserve(count); + for (u32 i = 0; i < count; ++i) { + u64 hash{}; + u8 age{}; + if (!file.read(reinterpret_cast<char*>(&hash), sizeof(hash)) || + !file.read(reinterpret_cast<char*>(&age), sizeof(age))) { + break; + } + if (age < UsedPipelinesMaxAge) { // lists saved with a longer window + used.emplace(hash, age); + } + } + return used; +} + +// Without a list of the pipelines recent sessions used (the first boot with this cache, or a session +// that ended before saving it), this many cached graphics pipelines at most are built at boot and the +// rest when first needed. A cache that grew over many builds held 48k pipelines, ~21 GB of driver +// memory built before the game started; the list is only saved at exit or after two minutes, so a +// boot that ran out of memory never saved one and the next boot did the same. +constexpr size_t MaxPipelinesWithoutUsedList = 3000; + +// EDEN_NX2_PRELOAD_ALL_PIPELINES=1 builds every cached pipeline at boot, as before. +bool PreloadAllPipelines() { + static const bool all = [] { + const char* value = std::getenv("EDEN_NX2_PRELOAD_ALL_PIPELINES"); + return value != nullptr && value[0] == '1'; + }(); + return all; +} +} // Anonymous namespace + PipelineCache::~PipelineCache() { + SaveUsedPipelines(); if (use_vulkan_pipeline_cache && !vulkan_pipeline_cache_filename.empty()) { SerializeVulkanPipelineCache(vulkan_pipeline_cache_filename, vulkan_pipeline_cache, CACHE_VERSION); } } +void PipelineCache::SaveUsedPipelines() { + used_pipelines_saved = std::chrono::steady_clock::now(); + // A session too short to say what the game uses (a boot to the menu) keeps the old list. + constexpr size_t MinimumUsed = 64; + if (used_pipelines_filename.empty() || + used_this_session.size() + used_and_evicted.size() < MinimumUsed) { + return; + } + std::unordered_map<u64, u8> next; + next.reserve(used_pipelines.size() + used_this_session.size()); + for (const auto& [key, pipeline] : graphics_cache) { + if (used_this_session.contains(pipeline.get())) { + next.emplace(key.Hash(), u8{0}); + } + } + for (const u64 hash : used_and_evicted) { + next.emplace(hash, u8{0}); + } + for (const auto& [hash, age] : used_pipelines) { + if (age + 1 < UsedPipelinesMaxAge) { + next.try_emplace(hash, static_cast<u8>(age + 1)); + } + } + std::ofstream file{used_pipelines_filename, std::ios::binary | std::ios::trunc}; + const u32 count = static_cast<u32>(next.size()); + file.write(reinterpret_cast<const char*>(&UsedPipelinesMagic), sizeof(UsedPipelinesMagic)); + file.write(reinterpret_cast<const char*>(&count), sizeof(count)); + for (const auto& [hash, age] : next) { + file.write(reinterpret_cast<const char*>(&hash), sizeof(hash)); + file.write(reinterpret_cast<const char*>(&age), sizeof(age)); + } +} + GraphicsPipeline* PipelineCache::CurrentGraphicsPipeline() { if (!RefreshStages(graphics_key.unique_hashes)) { @@ -582,6 +695,7 @@ GraphicsPipeline* PipelineCache::CurrentGraphicsPipeline() { GraphicsPipeline* const next{current_pipeline->Next(graphics_key)}; if (next) { current_pipeline = next; + current_pipeline->MarkUsed(frame_count); return BuiltPipeline(current_pipeline); } } @@ -629,6 +743,11 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading return; } pipeline_cache_filename = base_dir / "vulkan.bin"; + used_pipelines_filename = base_dir / "vulkan_used.bin"; + used_pipelines = LoadUsedPipelines(used_pipelines_filename); + used_pipelines_saved = std::chrono::steady_clock::now(); + size_t skipped_pipelines = 0; + size_t queued_graphics = 0; if (use_vulkan_pipeline_cache) { vulkan_pipeline_cache_filename = base_dir / "vulkan_pipelines.bin"; @@ -669,6 +788,15 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading GraphicsPipelineCacheKey key; file.read(reinterpret_cast<char*>(&key), sizeof(key)); + // Task/mesh pipelines are not stored (their mesh registers are not in the format); skip + // any that were. + if (std::ranges::any_of(envs, [](const FileEnvironment& env) { + return env.ShaderStage() == Shader::Stage::Task || + env.ShaderStage() == Shader::Stage::Mesh; + })) { + return; + } + if ((key.state.extended_dynamic_state != 0) != dynamic_features.has_extended_dynamic_state || (key.state.extended_dynamic_state_2 != 0) != @@ -697,6 +825,16 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading return; } + const u64 hash = key.Hash(); + disk_pipelines.insert(hash); + if (!PreloadAllPipelines() && + (used_pipelines.empty() ? queued_graphics >= MaxPipelinesWithoutUsedList + : !used_pipelines.contains(hash))) { + ++skipped_pipelines; + return; + } + ++queued_graphics; + workers.QueueWork([this, key, envs_ = std::move(envs), &state, &callback]() mutable { ShaderPools pools; boost::container::static_vector<Shader::Environment*, 5> env_ptrs; @@ -721,6 +859,11 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading load_graphics); LOG_INFO(Render_Vulkan, "Total Pipeline Count: {}", state.total); + if (skipped_pipelines != 0) { + LOG_INFO(Render_Vulkan, + "{} cached graphics pipelines are built when first needed", + skipped_pipelines); + } std::unique_lock lock{state.mutex}; callback(VideoCore::LoadCallbackStage::Build, 0, state.total); @@ -837,13 +980,66 @@ GraphicsPipeline* PipelineCache::CurrentGraphicsPipelineSlowPath() { if (!pipeline) { return nullptr; } + if (used_this_session.insert(pipeline.get()).second && + std::chrono::steady_clock::now() - used_pipelines_saved > std::chrono::minutes{2}) { + SaveUsedPipelines(); + } if (current_pipeline) { current_pipeline->AddTransition(pipeline.get()); } current_pipeline = pipeline.get(); + current_pipeline->MarkUsed(frame_count); return BuiltPipeline(current_pipeline); } +void PipelineCache::TickFrame() { + ++frame_count; + while (!evicted_pipelines.empty() && scheduler.IsFree(evicted_pipelines.front().first)) { + evicted_pipelines.pop_front(); + } + constexpr u64 EvictCheckFrames = 600; + if (frame_count % EvictCheckFrames == 0) { + EvictUnusedPipelines(); + } +} + +void PipelineCache::EvictUnusedPipelines() { + // Each built pipeline holds ~0.45 MB of driver memory and none was ever destroyed, so it grew + // with every place a session went (title 1: 1,500 on its title screen alone). One unused for + // this long is destroyed; it takes 2-3 ms to build again from the driver's pipeline cache. + constexpr u64 EvictAfterFrames = 9000; + if (frame_count <= EvictAfterFrames) { + return; + } + const u64 cutoff = frame_count - EvictAfterFrames; + const u64 tick = scheduler.CurrentTick(); + size_t evicted = 0; + for (auto it = graphics_cache.begin(); it != graphics_cache.end();) { + GraphicsPipeline* const pipeline = it->second.get(); + if (pipeline == nullptr || pipeline == current_pipeline || !pipeline->IsBuilt() || + pipeline->LastUse() >= cutoff) { + ++it; + continue; + } + if (used_this_session.erase(pipeline) != 0) { + used_and_evicted.insert(it->first.Hash()); + } + evicted_pipelines.emplace_back(tick, std::move(it->second)); + it = graphics_cache.erase(it); + ++evicted; + } + if (evicted == 0) { + return; + } + for (auto& [key, pipeline] : graphics_cache) { + if (pipeline) { + pipeline->ClearTransitions(); + } + } + LOG_INFO(Render_Vulkan, "Evicted {} graphics pipelines unused for {} frames, {} left", evicted, + EvictAfterFrames, graphics_cache.size()); +} + GraphicsPipeline* PipelineCache::BuiltPipeline(GraphicsPipeline* pipeline) const noexcept { if (pipeline->IsBuilt()) { return pipeline; @@ -985,7 +1181,12 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline( ++env_index; const u32 cfg_offset{static_cast<u32>(env.StartAddress() + sizeof(Shader::ProgramHeader))}; - Shader::Maxwell::Flow::CFG cfg(env, pools.flow_block, cfg_offset, index == 0); + try { + // Walks the program for the dump; the one that failed may fail here again, and an + // exception leaving this handler would end the process. + Shader::Maxwell::Flow::CFG cfg(env, pools.flow_block, cfg_offset, index == 0); + } catch (const Shader::Exception&) { + } env.Dump(hash, key.unique_hashes[index]); } LOG_ERROR(Render_Vulkan, "{}", exception.what()); @@ -1033,6 +1234,18 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline() { if (!pipeline || pipeline_cache_filename.empty()) { return pipeline; } + if (pipeline->IsMeshPipeline()) { + // Not stored: task and mesh programs are translated with the mesh registers, which the + // cache format does not carry. The few of them rebuild quickly from the driver cache. + QueueVulkanPipelineCacheFlush(); + return pipeline; + } + if (disk_pipelines.contains(graphics_key.Hash())) { + // In the disk cache already, only not built at boot. + QueueVulkanPipelineCacheFlush(); + return pipeline; + } + disk_pipelines.insert(graphics_key.Hash()); serialization_thread.QueueWork([this, key = graphics_key, envs = std::move(environments.envs)] { boost::container::static_vector<const GenericEnvironment*, Maxwell::MaxShaderProgram> env_ptrs; @@ -1055,7 +1268,8 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline( }; const GPUVAddr program_base{kepler_compute->regs.code_loc.Address()}; const auto& qmd{kepler_compute->launch_description}; - ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start}; + ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start, + shader->lowered_native}; env.SetCachedSize(shader->size_bytes); main_pools.ReleaseContents(); diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.h b/src/video_core/renderer_vulkan/vk_pipeline_cache.h index 71dfb46e3d..66fa615ea5 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.h +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.h @@ -10,7 +10,10 @@ #include <atomic> #include <chrono> #include <cstddef> +#include <deque> #include <filesystem> +#include <unordered_map> +#include <unordered_set> #include <memory> #include <type_traits> #include "common/container/unordered_map.h" @@ -114,6 +117,8 @@ public: [[nodiscard]] GraphicsPipeline* CurrentGraphicsPipeline(); + void TickFrame(); + [[nodiscard]] ComputePipeline* CurrentComputePipeline(); void LoadDiskResources(u64 title_id, std::stop_token stop_loading, @@ -163,6 +168,12 @@ private: GraphicsPipelineCacheKey graphics_key{}; GraphicsPipeline* current_pipeline{}; + /// Destroys graphics pipelines unused for a while, once the GPU is done with them. + void EvictUnusedPipelines(); + u64 frame_count{}; + std::deque<std::pair<u64, std::unique_ptr<GraphicsPipeline>>> evicted_pipelines; ///< (tick, pipeline) + std::unordered_set<u64> used_and_evicted; ///< Key hashes of pipelines used this session, evicted since + ::Common::unordered_map<ComputePipelineCacheKey, std::unique_ptr<ComputePipeline>> compute_cache; ::Common::unordered_map<GraphicsPipelineCacheKey, std::unique_ptr<GraphicsPipeline>> graphics_cache; @@ -173,6 +184,16 @@ private: std::filesystem::path pipeline_cache_filename; + /// Key hashes of the graphics pipelines in the disk cache. Only those a recent session used + /// (used_pipelines) are built at boot; the rest are built when first needed, and not stored + /// again. Each built pipeline holds driver memory (title 1: 6.5k cached, a quarter used). + std::unordered_set<u64> disk_pipelines; + std::unordered_map<u64, u8> used_pipelines; ///< Key hash -> sessions since it was last used + std::unordered_set<const GraphicsPipeline*> used_this_session; + std::filesystem::path used_pipelines_filename; + std::chrono::steady_clock::time_point used_pipelines_saved{}; + void SaveUsedPipelines(); + std::filesystem::path vulkan_pipeline_cache_filename; vk::PipelineCache vulkan_pipeline_cache; size_t pipelines_since_flush{}; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index cb31a0e7e4..d3dbba6163 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -9,6 +9,7 @@ #include <memory> #include <cstdio> #include <cstdlib> +#include <filesystem> #include <tuple> #include <set> #include <fstream> @@ -279,9 +280,37 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) { report_failure("no pipeline"); return; } + // Task/mesh pipelines: the draw's vertex count is the number of task workgroups. + mesh_draw = pipeline->IsMeshPipeline(); + // EDEN_NX2_NO_MESH=1: skip task/mesh draws as before they were supported (A/B comparisons). + // EDEN_NX2_NO_MESH_FILE=<path>: skip them while that file exists (checked every 1024 draws), + // to compare the same scene with and without them in one run. + static const bool no_mesh_always{std::getenv("EDEN_NX2_NO_MESH") != nullptr}; + static const char* const no_mesh_file{std::getenv("EDEN_NX2_NO_MESH_FILE")}; + static bool no_mesh{no_mesh_always}; + static u32 no_mesh_check{}; + if (no_mesh_file && (no_mesh_check++ % 1024) == 0) { + no_mesh = std::filesystem::exists(no_mesh_file); + } + if (mesh_draw && (no_mesh || !device.IsExtMeshShaderSupported())) { + report_failure(no_mesh ? "EDEN_NX2_NO_MESH" : "no VK_EXT_mesh_shader"); + return; + } std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; // update engine as channel may be different. pipeline->SetEngine(maxwell3d, gpu_memory); + const u32 repack_dirty = buffer_cache_runtime.SetVertexRepackLayout( + maxwell3d->regs, pipeline->HasDynamicVertexInput()); + if (repack_dirty) { + auto& flags = maxwell3d->dirty.flags; + flags[VideoCommon::Dirty::VertexBuffers] = true; + flags[Dirty::VertexInput] = true; + for (u32 binding = 0; binding < 32; ++binding) { + if (repack_dirty & (1U << binding)) { + flags[VideoCommon::Dirty::VertexBuffer0 + binding] = true; + } + } + } if (Profile::Scope scope{Profile::Zone::Configure}; !pipeline->Configure(is_indexed)) { report_failure("configure failed"); return; @@ -541,6 +570,12 @@ void RasterizerVulkan::Draw(bool is_indexed, u32 instance_count) { const u32 num_instances{instance_count}; const DrawParams draw_params{MakeDrawParams(draw_state, num_instances, is_indexed)}; + if (mesh_draw) { + scheduler.Record([groups = draw_params.num_vertices](vk::CommandBuffer cmdbuf) { + cmdbuf.DrawMeshTasksEXT(groups, 1, 1); + }); + return; + } scheduler.Record([draw_params](vk::CommandBuffer cmdbuf) { if (draw_params.is_indexed) { cmdbuf.DrawIndexed(draw_params.num_vertices, draw_params.num_instances, @@ -573,6 +608,14 @@ void RasterizerVulkan::DrawIndirect() { const auto& params = maxwell3d->draw_manager.indirect_state; buffer_cache.SetDrawIndirect(¶ms); PrepareDraw(params.is_indexed, [this, ¶ms] { + if (mesh_draw) { + // Indirect task/mesh draws would need vkCmdDrawMeshTasksIndirectEXT's layout. + static bool logged{}; + if (!std::exchange(logged, true)) { + LOG_WARNING(Render_Vulkan, "Indirect task/mesh draw skipped"); + } + return; + } const auto indirect_buffer = buffer_cache.GetDrawIndirectBuffer(); const auto& buffer = indirect_buffer.first; const auto& offset = indirect_buffer.second; @@ -1200,6 +1243,7 @@ void RasterizerVulkan::FlushCommands() { void RasterizerVulkan::TickFrame() { draw_counter = 0; + pipeline_cache.TickFrame(); guest_descriptor_queue.TickFrame(); compute_pass_descriptor_queue.TickFrame(); descriptor_buffer_ring.TickFrame(); @@ -1231,6 +1275,14 @@ bool RasterizerVulkan::AccelerateSurfaceCopy(const Tegra::Engines::Fermi2D::Surf return texture_cache.BlitImage(dst, src, copy_config); } +bool RasterizerVulkan::AccelerateTemporalUpscale( + const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, std::array<f32, 2> jitter, + std::array<f32, 2> motion_scale) { + std::scoped_lock lock{texture_cache.mutex}; + return texture_cache.TemporalUpscale(dst, src, motion_vectors, jitter, motion_scale); +} + Tegra::Engines::AccelerateDMAInterface& RasterizerVulkan::AccessAccelerateDMA() { return accelerate_dma; } @@ -2261,7 +2313,8 @@ void RasterizerVulkan::UpdateVertexInput(Tegra::Engines::Maxwell3D::Regs& regs) .location = index, .binding = binding, .format = MaxwellToVK::VertexFormat(device, attribute.type, attribute.size), - .offset = attribute.offset, + .offset = attribute.offset + + (buffer_cache_runtime.IsVertexRepacked(binding) && attribute.offset >= 65 ? 3 : 0), }); } @@ -2272,7 +2325,7 @@ void RasterizerVulkan::UpdateVertexInput(Tegra::Engines::Maxwell3D::Regs& regs) .sType = VK_STRUCTURE_TYPE_VERTEX_INPUT_BINDING_DESCRIPTION_2_EXT, .pNext = nullptr, .binding = binding, - .stride = input_binding.stride, + .stride = buffer_cache_runtime.IsVertexRepacked(binding) ? 72U : input_binding.stride.Value(), .inputRate = is_instanced ? VK_VERTEX_INPUT_RATE_INSTANCE : VK_VERTEX_INPUT_RATE_VERTEX, .divisor = is_instanced ? input_binding.frequency : 1, }); diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index ae9929ca0b..fc282fbe8d 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -128,6 +128,11 @@ public: bool AccelerateSurfaceCopy(const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& dst, const Tegra::Engines::Fermi2D::Config& copy_config) override; + bool AccelerateTemporalUpscale(const Tegra::Engines::Fermi2D::Surface& src, + const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, + std::array<f32, 2> jitter, + std::array<f32, 2> motion_scale) override; Tegra::Engines::AccelerateDMAInterface& AccessAccelerateDMA() override; void AccelerateInlineToMemory(GPUVAddr address, size_t copy_size, std::span<const u8> memory) override; @@ -230,6 +235,7 @@ private: boost::container::static_vector<VkSampler, MAX_TEXTURES> sampler_handles; u32 draw_counter = 0; + bool mesh_draw{}; ///< the draw being prepared uses a task/mesh pipeline }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp index 7b6571d33d..98787b9add 100644 --- a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp +++ b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp @@ -36,11 +36,25 @@ constexpr VkDeviceSize MAX_ALIGNMENT = 256; #if defined(__FreeBSD__) constexpr VkDeviceSize MAX_STREAM_BUFFER_SIZE = 128_MiB; #else -constexpr VkDeviceSize MAX_STREAM_BUFFER_SIZE = 256_MiB; +// 128 MiB on Windows too: 256 MiB sat mostly unused in device memory on Switch 2 titles. +constexpr VkDeviceSize MAX_STREAM_BUFFER_SIZE = 128_MiB; #endif size_t GetStreamBufferSize(const Device& device) { if (!device.HasDebuggingToolAttached()) { + if (device.IsIntegrated()) { + // An integrated GPU's device-local host-visible heap can be no bigger than the stream + // buffer (Radeon 780M: 256 MiB, 243 MiB budget), which then sat over that budget. + // Leave a quarter of it free. + VkDeviceSize heap_size = 0; + ForEachDeviceLocalHostVisibleHeap( + device, [&heap_size](size_t, VkMemoryHeap& heap) { + heap_size = (std::max)(heap_size, heap.size); + }); + if (heap_size != 0 && heap_size <= MAX_STREAM_BUFFER_SIZE) { + return Common::AlignUp(heap_size * 3 / 4, MAX_ALIGNMENT); + } + } return MAX_STREAM_BUFFER_SIZE; } diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index ddd55de690..30981fe413 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -244,6 +244,35 @@ enum class SparseTilesMode { return size >= (32U << 20) && Tegra::Nx2InSparseRegion(gpu_addr, size); } +// Vulkan's standard sparse block shape for single-sampled 2D images (64 KiB blocks), in texel +// blocks, for a texel block of `block_bytes`. Only valid when the device reports +// residencyStandard2DBlockShape. +[[nodiscard]] bool StandardSparseBlockShape(const Device& device, u64 block_bytes, + VkExtent3D& shape) { + if (!device.GetPhysical().GetProperties().sparseProperties.residencyStandard2DBlockShape) { + return false; + } + switch (block_bytes) { + case 1: + shape = {256, 256, 1}; + return true; + case 2: + shape = {256, 128, 1}; + return true; + case 4: + shape = {128, 128, 1}; + return true; + case 8: + shape = {128, 64, 1}; + return true; + case 16: + shape = {64, 64, 1}; + return true; + default: + return false; + } +} + // The most device memory one SparseLayers image binds: 768 MiB, 384 MiB on GPUs with less than // 7 GiB (title 1's BC6H arrays are 85 MB a layer). [[nodiscard]] VkDeviceSize SparseLayersCap(const Device& device) { @@ -304,6 +333,20 @@ constexpr std::array<VkFormat, 2> OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM info.size.depth == 1; } +// The most a trimmed ASTC array keeps on the host: 512 MiB, 256 MiB on GPUs with less than 7 GiB +// (title 1's two terrain arrays took 724 MiB of a 6 GB card next to ~1.2 GB of other sparse terrain +// textures). EDEN_NX2_ASTC_ARRAY_MIB=<MiB> sets it. +[[nodiscard]] u64 TrimmedAstcArrayLimit(const Device& device) { + static const u64 forced = [] { + const char* value = std::getenv("EDEN_NX2_ASTC_ARRAY_MIB"); + return value != nullptr ? std::strtoull(value, nullptr, 10) << 20 : 0ULL; + }(); + if (forced != 0) { + return forced; + } + return device.GetDeviceLocalMemory() < (u64{7} << 30) ? (u64{256} << 20) : (u64{512} << 20); +} + // The host shape of an image whose decoded copy would not fit in VRAM: a GPU-decoded ASTC array // over 1 GiB as RGBA8 (title 1's 8192x8192x68 ASTC 8x8 virtual textures are 24 GB each) keeps only // its mip levels from `level_offset` on, so host mip 0 is guest mip `level_offset`. Normalized @@ -324,8 +367,9 @@ constexpr std::array<VkFormat, 2> OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM if (decoded_bytes(0) <= (u64{1} << 30)) { return info; } + const u64 limit = TrimmedAstcArrayLimit(device); while (level_offset + 1 < static_cast<u32>(info.resources.levels) && - decoded_bytes(level_offset) > (u64{512} << 20)) { + decoded_bytes(level_offset) > limit) { ++level_offset; } ImageInfo host = info; @@ -1491,6 +1535,154 @@ void TextureCacheRuntime::ReinterpretImage(Image& dst, Image& src, }); } +bool TextureCacheRuntime::TemporalUpscale(Framebuffer* dst_framebuffer, ImageView& dst, + ImageView& input_color, ImageView& motion_vectors, + std::array<f32, 2> jitter, + std::array<f32, 2> motion_scale) { + if (dst.Samples() != VK_SAMPLE_COUNT_1_BIT || ImageAspectMask(dst.format) != VK_IMAGE_ASPECT_COLOR_BIT) { + return false; + } + const VkFormat format = + MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, false, dst.format).format; + const VkExtent2D extent{dst.size.width, dst.size.height}; + const VkImage output = dst.ImageHandle(); + bool reset = false; + if (!temporal_history.image || temporal_history.format != format || + temporal_history.extent.width != extent.width || + temporal_history.extent.height != extent.height) { + if (temporal_history.image) { + scheduler.Finish(); // the old history may still be sampled + } + temporal_history.image = memory_allocator.CreateImage(VkImageCreateInfo{ + .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .imageType = VK_IMAGE_TYPE_2D, + .format = format, + .extent = {extent.width, extent.height, 1}, + .mipLevels = 1, + .arrayLayers = 1, + .samples = VK_SAMPLE_COUNT_1_BIT, + .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, + }); + temporal_history.view = device.GetLogical().CreateImageView(VkImageViewCreateInfo{ + .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .image = *temporal_history.image, + .viewType = VK_IMAGE_VIEW_TYPE_2D, + .format = format, + .components{}, + .subresourceRange{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = 1, + }, + }); + temporal_history.format = format; + temporal_history.extent = extent; + reset = true; + LOG_INFO(Render_Vulkan, "Temporal upscale into {}x{} (format {})", extent.width, extent.height, + static_cast<u32>(format)); + } + + // The history holds this pass's previous result, copied right after drawing it: the game may + // draw into its DLSS output afterwards (post-processing), which must not feed back. The history + // stays in the GENERAL layout, which the blit helper's descriptors use. + const VkImage history = *temporal_history.image; + const VkImageSubresourceLayers output_layers{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .mipLevel = static_cast<u32>(dst.range.base.level), + .baseArrayLayer = static_cast<u32>(dst.range.base.layer), + .layerCount = 1, + }; + static constexpr VkImageSubresourceLayers history_layers{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .mipLevel = 0, + .baseArrayLayer = 0, + .layerCount = 1, + }; + const auto barrier = [](VkImage image, VkAccessFlags src_access, VkAccessFlags dst_access, + VkImageLayout old_layout, const VkImageSubresourceLayers& layers) { + return VkImageMemoryBarrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = src_access, + .dstAccessMask = dst_access, + .oldLayout = old_layout, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = image, + .subresourceRange{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = layers.mipLevel, + .levelCount = 1, + .baseArrayLayer = layers.baseArrayLayer, + .layerCount = 1, + }, + }; + }; + if (reset) { + // A new history image has no contents yet; the first frame does not read it. + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([history, barrier](vk::CommandBuffer cmdbuf) { + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, + 0, barrier(history, 0, VK_ACCESS_SHADER_READ_BIT, + VK_IMAGE_LAYOUT_UNDEFINED, history_layers)); + }); + } + const Region2D dst_region{ + .start = {0, 0}, + .end = {static_cast<s32>(extent.width), static_cast<s32>(extent.height)}, + }; + // Each frame weighs at most 0.1 against the history (scaled by how close its samples land to + // the pixel; the neighbourhood clip limits what older samples contribute). + constexpr f32 current_weight = 0.1f; + blit_image_helper.TemporalUpscale(dst_framebuffer, input_color, motion_vectors, + *temporal_history.view, dst_region, jitter, motion_scale, + reset ? 1.0f : current_weight); + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([output, history, extent, output_layers, barrier](vk::CommandBuffer cmdbuf) { + constexpr VkAccessFlags written = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + const std::array before{ + barrier(output, written, VK_ACCESS_TRANSFER_READ_BIT, VK_IMAGE_LAYOUT_GENERAL, + output_layers), + barrier(history, VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, + VK_IMAGE_LAYOUT_GENERAL, history_layers), + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, {}, before); + const VkImageCopy copy{ + .srcSubresource = output_layers, + .srcOffset = {0, 0, 0}, + .dstSubresource = history_layers, + .dstOffset = {0, 0, 0}, + .extent = {extent.width, extent.height, 1}, + }; + cmdbuf.CopyImage(output, VK_IMAGE_LAYOUT_GENERAL, history, VK_IMAGE_LAYOUT_GENERAL, copy); + const std::array after{ + barrier(output, VK_ACCESS_TRANSFER_READ_BIT, + VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | + VK_ACCESS_TRANSFER_READ_BIT, + VK_IMAGE_LAYOUT_GENERAL, output_layers), + barrier(history, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT, + VK_IMAGE_LAYOUT_GENERAL, history_layers), + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, + {}, {}, after); + }); + return true; +} + void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst, ImageView& src, const Region2D& dst_region, const Region2D& src_region, Tegra::Engines::Fermi2D::Filter filter, @@ -2026,10 +2218,22 @@ u64 TextureCacheRuntime::GetDeviceMemoryUsage() const { return device.GetDeviceMemoryUsage(); } +u64 TextureCacheRuntime::CurrentTick() const noexcept { + return scheduler.CurrentTick(); +} + +bool TextureCacheRuntime::IsFree(u64 tick) const noexcept { + return scheduler.IsFree(tick); +} + u64 TextureCacheRuntime::GetDeviceMemoryBudget() const { return device.GetDeviceMemoryBudget(); } +u64 TextureCacheRuntime::GetDeviceMemorySlack() const { + return device.GetDeviceMemorySlack(); +} + bool TextureCacheRuntime::CanDownloadMsaa(const VideoCommon::ImageInfo& info) const { const VkImageAspectFlags aspect_mask = ImageAspectMask(info.format); if ((aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) != 0) { @@ -2107,10 +2311,33 @@ void TextureCacheRuntime::ReleaseMsaaScratchImage(VkImage image) { } void TextureCacheRuntime::TickFrame() { + static constexpr u32 MAX_UNUSED_SCRATCH_FRAMES = 60; + // Finished decode images wait for the next texture of their shape, up to this much memory. + static constexpr u64 MaxFreeTranscodeBytes = u64{256} << 20; + const auto decode_bytes = [](const PendingTranscode& t) { + u64 bytes{}; + for (u32 level = 0; level < t.levels; ++level) { + bytes += u64{(std::max)(t.width >> level, 1U)} * (std::max)(t.height >> level, 1U) * 4; + } + return bytes * t.layers; + }; + u64 free_bytes{}; + for (const PendingTranscode& t : free_transcodes) { + free_bytes += decode_bytes(t); + } while (!pending_transcodes.empty() && scheduler.IsFree(pending_transcodes.front().tick)) { + PendingTranscode& done = pending_transcodes.front(); + const u64 bytes = decode_bytes(done); + if (free_bytes + bytes <= MaxFreeTranscodeBytes) { + done.unused_frames = 0; + free_bytes += bytes; + free_transcodes.push_back(std::move(done)); + } pending_transcodes.pop_front(); } - static constexpr u32 MAX_UNUSED_SCRATCH_FRAMES = 60; + std::erase_if(free_transcodes, [](PendingTranscode& t) { + return ++t.unused_frames > MAX_UNUSED_SCRATCH_FRAMES; + }); std::erase_if(msaa_scratch_images, [this](MsaaScratchImage& scratch) { if (!scheduler.IsFree(scratch.tick)) { scratch.unused_frames = 0; @@ -2650,7 +2877,24 @@ bool Image::InitSparseLayers() { const u32 block_width = huge_astc ? 1 : VideoCore::Surface::DefaultBlockWidth(info.format); const u32 block_height = huge_astc ? 1 : VideoCore::Surface::DefaultBlockHeight(info.format); const u64 block_bytes = huge_astc ? 4 : VideoCore::Surface::BytesPerBlock(info.format); - const VkExtent3D granularity = it->formatProperties.imageGranularity; + VkExtent3D granularity = it->formatProperties.imageGranularity; + if (granularity.width == 0 || granularity.height == 0) { + // Seen with AMD's driver: the sparse block size comes back as 0, and the size computation + // below divided by it. Use the standard 2D block shape when the device promises it. + if (!StandardSparseBlockShape(device, block_bytes, granularity)) { + LOG_WARNING(Render_Vulkan, + "Sparse layered image {:#x}: the driver reports no sparse block size and " + "no standard block shape; not using sparse layers", + gpu_addr); + return false; + } + granularity.width *= block_width; + granularity.height *= block_height; + LOG_INFO(Render_Vulkan, + "Sparse layered image {:#x}: no block size from the driver, using the standard " + "{}x{} texel block", + gpu_addr, granularity.width, granularity.height); + } s.tile_width = granularity.width; s.tile_height = granularity.height; const u64 tile = s.requirements.alignment; @@ -4087,30 +4331,47 @@ void TextureCacheRuntime::TranscodeAstcToBcn( const u32 levels = static_cast<u32>(info.resources.levels); const u32 layers = static_cast<u32>(info.resources.layers); constexpr VkFormat DecodeFormat = VK_FORMAT_A8B8G8R8_UNORM_PACK32; - PendingTranscode pending{ - .tick = 0, - .image = memory_allocator.CreateImage(VkImageCreateInfo{ - .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - .imageType = VK_IMAGE_TYPE_2D, - .format = DecodeFormat, - .extent = {width, height, 1}, - .mipLevels = levels, - .arrayLayers = layers, - .samples = VK_SAMPLE_COUNT_1_BIT, - .tiling = VK_IMAGE_TILING_OPTIMAL, - .usage = VK_IMAGE_USAGE_STORAGE_BIT, - .sharingMode = VK_SHARING_MODE_EXCLUSIVE, - .queueFamilyIndexCount = 0, - .pQueueFamilyIndices = nullptr, - .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, - }, true), - .views = {}, - }; - for (u32 level = 0; level < levels; ++level) { - pending.views.push_back( - MakeStorageView(device.GetLogical(), level, *pending.image, DecodeFormat)); + // A decode image of the same shape the GPU is done with: title 1's title screen decodes + // dozens of textures a frame, mostly of a few shapes. Making one took 263 us; from VMA's + // shared blocks 90 us, but freeing them there left ~1 GB of those blocks unused. + const auto reusable = std::ranges::find_if(free_transcodes, [&](const PendingTranscode& t) { + return t.width == width && t.height == height && t.levels == levels && t.layers == layers; + }); + PendingTranscode pending; + if (reusable != free_transcodes.end()) { + pending = std::move(*reusable); + free_transcodes.erase(reusable); + } else { + pending = PendingTranscode{ + .tick = 0, + .image = memory_allocator.CreateImage(VkImageCreateInfo{ + .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .imageType = VK_IMAGE_TYPE_2D, + .format = DecodeFormat, + .extent = {width, height, 1}, + .mipLevels = levels, + .arrayLayers = layers, + .samples = VK_SAMPLE_COUNT_1_BIT, + .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = VK_IMAGE_USAGE_STORAGE_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, + }, true), + .views = {}, + .width = width, + .height = height, + .levels = levels, + .layers = layers, + .unused_frames = 0, + }; + for (u32 level = 0; level < levels; ++level) { + pending.views.push_back( + MakeStorageView(device.GetLogical(), level, *pending.image, DecodeFormat)); + } } astc_decoder_pass->AssembleInto( info, 0, image.guest_size_bytes, @@ -4119,7 +4380,7 @@ void TextureCacheRuntime::TranscodeAstcToBcn( .aspect_mask = VK_IMAGE_ASPECT_COLOR_BIT, .initialized = false, .level_view = [&pending](s32 level) { return *pending.views[level]; }, - .wait = false, + .wait = true, }, map, swizzles); diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 43218d6ad3..564b2cbf9e 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -67,10 +67,22 @@ public: u64 GetDeviceMemoryUsage() const; + /// GPU progress: the texture cache frees a deleted image once the GPU is past its last use. + [[nodiscard]] u64 CurrentTick() const noexcept; + + [[nodiscard]] bool IsFree(u64 tick) const noexcept; + u64 GetDeviceMemoryBudget() const; + u64 GetDeviceMemorySlack() const; + bool CanReportMemoryUsage() const; + /// Images that did not fit the device memory budget so far (see MemoryAllocator). + u32 MemoryPressureEvents() const noexcept { + return memory_allocator.MemoryPressureEvents(); + } + void LogMemoryStatistics() const; /// True when ASTC is re-compressed to BC3 and opaque images may use BC1 instead. @@ -87,6 +99,12 @@ public: Tegra::Engines::Fermi2D::Filter filter, Tegra::Engines::Fermi2D::Operation operation); + /// Stands in for a DLSS evaluation: temporal upscale of input_color into dst, accumulated with + /// a copy of dst's previous contents (the last output). False when dst cannot take it. + bool TemporalUpscale(Framebuffer* dst_framebuffer, ImageView& dst, ImageView& input_color, + ImageView& motion_vectors, std::array<f32, 2> jitter, + std::array<f32, 2> motion_scale); + void CopyImage(Image& dst, Image& src, std::span<const VideoCommon::ImageCopy> copies); void CopyImageMSAA(Image& dst, Image& src, std::span<const VideoCommon::ImageCopy> copies); @@ -169,6 +187,15 @@ public: StagingBufferPool& staging_buffer_pool; BlitImageHelper& blit_image_helper; RenderPassCache& render_pass_cache; + + /// TemporalUpscale's history: its previous result, copied right after drawing it. + struct TemporalHistory { + vk::Image image; + vk::ImageView view; + VkFormat format{}; + VkExtent2D extent{}; + } temporal_history; + std::optional<ASTCDecoderPass> astc_decoder_pass; std::optional<BcEncoderPass> bc_encoder_pass; @@ -177,13 +204,20 @@ public: void TranscodeAstcToBcn(Image& image, const StagingBufferRef& map, std::span<const VideoCommon::SwizzleParameters> swizzles); - /// Temporary decode images, freed once the GPU is past `tick`. + /// Temporary decode images, reusable once the GPU is past `tick`. struct PendingTranscode { u64 tick; vk::Image image; std::vector<vk::ImageView> views; + u32 width; + u32 height; + u32 levels; + u32 layers; + u32 unused_frames; }; std::deque<PendingTranscode> pending_transcodes; + /// Decode images the GPU is done with, for the next texture of the same shape. + std::vector<PendingTranscode> free_transcodes; std::optional<BlockLinearUnswizzle3DPass> bl3d_unswizzle_pass; const Settings::ResolutionScalingInfo& resolution; @@ -375,6 +409,11 @@ public: return (this->*current_image).UsageFlags(); } + /// Where the image's memory lies, for the collector (TextureCache::RunGarbageCollector). + [[nodiscard]] vk::Image::MemoryBlock MemoryBlock() const noexcept { + return original_image.GetMemoryBlock(); + } + /// Returns true when the image is already initialized and mark it as initialized [[nodiscard]] bool ExchangeInitialization() noexcept { return std::exchange(initialized, true); diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.cpp b/src/video_core/renderer_vulkan/vk_update_descriptor.cpp index 742a8d9969..29f3ee71a1 100644 --- a/src/video_core/renderer_vulkan/vk_update_descriptor.cpp +++ b/src/video_core/renderer_vulkan/vk_update_descriptor.cpp @@ -20,10 +20,10 @@ UpdateDescriptorQueue::UpdateDescriptorQueue(const Device& device_, size_t frame bool supports_descriptor_buffer_) : device{device_}, frame_payload_size{frame_payload_size_}, supports_descriptor_buffer{supports_descriptor_buffer_}, - payload(frame_payload_size_ * FRAMES_IN_FLIGHT) + payload(std::make_unique_for_overwrite<DescriptorUpdateEntry[]>(frame_payload_size_ * FRAMES_IN_FLIGHT)) { - payload_start = payload.data(); - payload_cursor = payload.data(); + payload_start = payload.get(); + payload_cursor = payload.get(); } UpdateDescriptorQueue::~UpdateDescriptorQueue() = default; @@ -32,7 +32,7 @@ void UpdateDescriptorQueue::TickFrame() { if (++frame_index >= FRAMES_IN_FLIGHT) { frame_index = 0; } - payload_start = payload.data() + frame_index * frame_payload_size; + payload_start = payload.get() + frame_index * frame_payload_size; payload_cursor = payload_start; } diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.h b/src/video_core/renderer_vulkan/vk_update_descriptor.h index 9f30e65dbd..4d393376b4 100644 --- a/src/video_core/renderer_vulkan/vk_update_descriptor.h +++ b/src/video_core/renderer_vulkan/vk_update_descriptor.h @@ -6,6 +6,7 @@ #pragma once +#include <memory> #include <variant> #include <vector> #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -122,7 +123,7 @@ private: DescriptorUpdateEntry* payload_cursor = nullptr; DescriptorUpdateEntry* payload_start = nullptr; const DescriptorUpdateEntry* upload_start = nullptr; - std::vector<DescriptorUpdateEntry> payload; + std::unique_ptr<DescriptorUpdateEntry[]> payload; }; // TODO: should these be separate classes instead? diff --git a/src/video_core/shader_cache.cpp b/src/video_core/shader_cache.cpp index dfe80abac3..5faed0dba9 100644 --- a/src/video_core/shader_cache.cpp +++ b/src/video_core/shader_cache.cpp @@ -14,6 +14,7 @@ #include <boost/container/small_vector.hpp> #include "common/assert.h" +#include "common/cityhash.h" #include "shader_recompiler/frontend/maxwell/control_flow.h" #include "shader_recompiler/exception.h" #include "shader_recompiler/object_pool.h" @@ -48,6 +49,22 @@ void ShaderCache::SyncGuestHost() { ShaderCache::ShaderCache(Tegra::MaxwellDeviceMemoryManager& device_memory_) : device_memory{device_memory_} {} +// Task and mesh programs are translated with the mesh registers and the header of the pass-through +// geometry program (per-primitive outputs), so pipelines are keyed by them too. +u64 ShaderCache::MeshStateHash() const { + using Regs = Tegra::Engines::Maxwell3D::Regs; + const auto& regs{maxwell3d->regs}; + std::array<u32, 3 + 20> state{regs.reg_array[Regs::MeshShaderA], regs.reg_array[Regs::MeshShaderB], + regs.reg_array[Regs::MeshInitShader]}; + if (regs.IsShaderConfigEnabled(Regs::ShaderType::Geometry)) { + const auto& geometry{regs.pipelines[static_cast<size_t>(Regs::ShaderType::Geometry)]}; + const GPUVAddr address{(static_cast<u64>(geometry.reserved[0] & 0xff) << 32) | + geometry.reserved[1]}; + gpu_memory->ReadBlockUnsafe(address, state.data() + 3, 20 * sizeof(u32)); + } + return Common::CityHash64(reinterpret_cast<const char*>(state.data()), sizeof(state)) | 1; +} + bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) try { auto& dirty{maxwell3d->dirty.flags}; if (!dirty[VideoCommon::Dirty::Shaders]) { @@ -57,6 +74,7 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) try { const GPUVAddr base_addr{maxwell3d->regs.program_region.Address()}; const bool native = maxwell3d->GetNativeMacroEngine().native_turing; + const bool mesh{native && maxwell3d->regs.IsMeshEnabled()}; for (size_t index = 0; index < Tegra::Engines::Maxwell3D::Regs::MaxShaderProgram; ++index) { if (!maxwell3d->regs.IsShaderConfigEnabled(index)) { unique_hashes[index] = 0; @@ -69,6 +87,11 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) try { unique_hashes[index] = 0; continue; } + if (mesh && program == Tegra::Engines::Maxwell3D::Regs::ShaderType::Geometry) { + // A header without code: the mesh program's environment reads its output map. + unique_hashes[index] = 0; + continue; + } const GPUVAddr shader_addr{native ? (static_cast<u64>(shader_config.reserved[0] & 0xff) << 32) | shader_config.reserved[1] : base_addr + shader_config.offset}; @@ -89,7 +112,8 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) try { if (!shader_info) { const u32 start_address{native ? 0 : shader_config.offset}; try { - // Native programs are lowered while the environment is built. + // Native programs are lowered while the environment is built (once: the + // ShaderInfo keeps the result for the pipeline's environments). RethrowIfRejected(*cpu_shader_addr); GraphicsEnvironment env{*maxwell3d, *gpu_memory, program, native ? shader_addr : base_addr, start_address}; shader_info = MakeShaderInfo(env, *cpu_shader_addr); @@ -111,6 +135,10 @@ bool ShaderCache::RefreshStages(std::array<u64, 6>& unique_hashes) try { } shader_infos[index] = shader_info; unique_hashes[index] = shader_info->unique_hash; + if (mesh && (program == Tegra::Engines::Maxwell3D::Regs::ShaderType::VertexB || + program == Tegra::Engines::Maxwell3D::Regs::ShaderType::Tessellation)) { + unique_hashes[index] ^= MeshStateHash(); + } } last_shaders_valid = true; return true; @@ -202,7 +230,8 @@ void ShaderCache::GetGraphicsEnvironments(GraphicsEnvironments& result, const GPUVAddr address = native ? (static_cast<u64>(config.reserved[0] & 0xff) << 32) | config.reserved[1] : base_addr; - env = GraphicsEnvironment{*maxwell3d, *gpu_memory, program, address, start_address}; + env = GraphicsEnvironment{*maxwell3d, *gpu_memory, program, address, start_address, + shader_infos[index]->lowered_native}; env.SetCachedSize(shader_infos[index]->size_bytes); result.env_ptrs[env_index++] = &env; } @@ -342,6 +371,7 @@ const ShaderInfo* ShaderCache::MakeShaderInfo(GenericEnvironment& env, VAddr cpu info->unique_hash = env.CalculateHash(); info->size_bytes = env.ReadSizeBytes(); } + info->lowered_native = env.LoweredNative(); const size_t size_bytes{env.SourceSizeBytes() ? env.SourceSizeBytes() : info->size_bytes}; const ShaderInfo* const result{info.get()}; Register(std::move(info), cpu_addr, size_bytes); diff --git a/src/video_core/shader_cache.h b/src/video_core/shader_cache.h index f4179dd435..281a90cb6f 100644 --- a/src/video_core/shader_cache.h +++ b/src/video_core/shader_cache.h @@ -38,6 +38,8 @@ class GenericEnvironment; struct ShaderInfo { u64 unique_hash{}; size_t size_bytes{}; + /// Lowered code of a native program, reused when its pipelines are built. + std::shared_ptr<const LoweredNativeCode> lowered_native; }; class ShaderCache : public VideoCommon::ChannelSetupCaches<VideoCommon::ChannelInfo> { @@ -151,6 +153,8 @@ private: bool IsRejected(VAddr addr); /// Remembers the exception being handled as this address's rejection. void RememberRejection(VAddr addr); + /// Hash of the state task and mesh programs are translated with. + u64 MeshStateHash() const; Tegra::MaxwellDeviceMemoryManager& device_memory; diff --git a/src/video_core/shader_environment.cpp b/src/video_core/shader_environment.cpp index 9d189ad4f0..22e661c849 100644 --- a/src/video_core/shader_environment.cpp +++ b/src/video_core/shader_environment.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <algorithm> +#include <array> #include <chrono> #include <cstdlib> #include <thread> @@ -315,29 +316,107 @@ std::optional<u64> GenericEnvironment::TryFindSize() { Tegra::Texture::TICEntry GenericEnvironment::ReadTextureInfo(GPUVAddr tic_addr, u32 tic_limit, bool via_header_index, u32 raw) { const auto handle{Tegra::Texture::TexturePair(raw, via_header_index)}; - ASSERT(handle.first <= tic_limit); - const GPUVAddr descriptor_addr{tic_addr + handle.first * sizeof(Tegra::Texture::TICEntry)}; + const u32 index{Tegra::Texture::TextureHeaderIndex(handle.first, tic_limit)}; + ASSERT(index <= tic_limit); + const GPUVAddr descriptor_addr{tic_addr + index * sizeof(Tegra::Texture::TICEntry)}; Tegra::Texture::TICEntry entry; gpu_memory->ReadBlock(descriptor_addr, &entry, sizeof(entry)); return entry; } -void GenericEnvironment::InitializeNativeCode(u32 source_header_size) { +namespace { + +// Reads a native program a 4 KiB page at a time instead of one 16-byte instruction at a time. +// The reads do not flush the GPU caches first (ReadBlockUnsafe): the previous per-instruction +// ReadBlock did, once per instruction. An instruction that is not inside a fully mapped page is +// left to the caller. +class NativeInstructionReader { +public: + using Instruction = Shader::Ampere::Instruction; + static constexpr GPUVAddr PageSize = 0x1000; + + explicit NativeInstructionReader(Tegra::MemoryManager& gpu_memory_) + : gpu_memory{gpu_memory_} {} + + /// Copies the instruction at `address` into `out`, loading its page first if needed. + /// Returns false when the instruction is not inside a fully mapped page. + bool TryRead(GPUVAddr address, Instruction& out) { + if (!Holds(address)) { + LoadFrom(address); + } + if (!Holds(address)) { + return false; + } + out = page[(address - page_begin) / sizeof(Instruction)]; + return true; + } + +private: + bool Holds(GPUVAddr address) const { + return address >= page_begin && address + sizeof(Instruction) <= page_end; + } + + // Loads from `address` up to the next page boundary; loads nothing if that range is not + // fully mapped. + void LoadFrom(GPUVAddr address) { + const GPUVAddr next_page = (address | (PageSize - 1)) + 1; + const GPUVAddr length = next_page - address; + page_begin = address; + page_end = address; + if (length >= sizeof(Instruction) && gpu_memory.IsFullyMappedRange(address, length)) { + gpu_memory.ReadBlockUnsafe(address, page.data(), length); + page_end = next_page; + } + } + + Tegra::MemoryManager& gpu_memory; + std::array<Instruction, PageSize / sizeof(Instruction)> page; + GPUVAddr page_begin{}; + GPUVAddr page_end{}; +}; + +} // Anonymous namespace + +// A native program is captured and lowered once, when the shader cache first meets it. The +// environments built later for its pipelines receive that result instead of lowering again. +void GenericEnvironment::InitializeNativeCode(std::shared_ptr<const LoweredNativeCode> lowered, + u32 source_header_size) { + if (!lowered) { + lowered = LowerNativeCode(source_header_size); + } + if (lowered->texture_bound != ~0U) { + // Native TEX carries its cbuf index explicitly; TEXS obtains it from the environment. + texture_bound = lowered->texture_bound; + } + native_source_size = lowered->source_size; + code = lowered->code; + cached_lowest = 0; + cached_highest = static_cast<u32>((code.size() - 1) * sizeof(u64)); + lowered_native = std::move(lowered); +} + +std::shared_ptr<const LoweredNativeCode> GenericEnvironment::LowerNativeCode(u32 source_header_size) { using Shader::Ampere::Instruction; const u32 native_header_size = source_header_size; constexpr u32 maximum_instructions = Shader::Ampere::ProgramCaptureState::MaximumInstructions; std::vector<Instruction> native; Shader::Ampere::ProgramCaptureState capture; + NativeInstructionReader reader{*gpu_memory}; for (u32 index = 0; index < maximum_instructions; ++index) { const GPUVAddr address = program_base + native_header_size + index * sizeof(Instruction); - if (!gpu_memory->IsFullyMappedRange(address, sizeof(Instruction))) { - // The program may end before a branch target past its last EXIT (the capture can't - // tell); stop at the mapping's end and let the lowering judge what was captured. - if (!native.empty()) break; - throw Shader::RuntimeError("Unmapped native shader instruction at {:#x}", address); + Instruction instruction; + if (!reader.TryRead(address, instruction)) { + // Not inside a fully mapped page: check and read this one instruction on its own. + if (!gpu_memory->IsFullyMappedRange(address, sizeof(Instruction))) { + // The program may end before a branch target past its last EXIT (the capture + // can't tell); stop at the mapping's end and let the lowering judge what was + // captured. + if (!native.empty()) break; + throw Shader::RuntimeError("Unmapped native shader instruction at {:#x}", address); + } + gpu_memory->ReadBlockUnsafe(address, &instruction, sizeof(instruction)); } - auto& instruction = native.emplace_back(); - gpu_memory->ReadBlock(address, &instruction, sizeof(instruction)); + native.push_back(instruction); const auto status = capture.Observe(instruction, index); if (status == Shader::Ampere::ProgramCaptureState::Result::InvalidTarget) { if (const char* dir = std::getenv("EDEN_NX2_SHADER_DUMP")) { @@ -439,39 +518,46 @@ void GenericEnvironment::InitializeNativeCode(u32 source_header_size) { program_base, lowered.error_offset, lowered.error, words[0] & 0xfff, words[0], words[1], words[2], words[3]); } - if (lowered.required_texture_buffer != ~0U) { - // Native TEX carries its cbuf index explicitly; TEXS obtains it from the environment. - texture_bound = lowered.required_texture_buffer; - } - native_source_size = native_header_size + lowered.source_size; - code.resize(sizeof(sph) / sizeof(u64)); - std::memcpy(code.data(), &sph, sizeof(sph)); + auto result = std::make_shared<LoweredNativeCode>(); + result->texture_bound = lowered.required_texture_buffer; + result->source_size = native_header_size + lowered.source_size; + auto& lowered_code = result->code; + lowered_code.resize(sizeof(sph) / sizeof(u64)); + std::memcpy(lowered_code.data(), &sph, sizeof(sph)); for (const u64 instruction : lowered.instructions) { - if ((code.size() * sizeof(u64)) % 32 == 0) { - code.push_back(0); // Maxwell scheduling word, absent from native instructions. + if ((lowered_code.size() * sizeof(u64)) % 32 == 0) { + lowered_code.push_back(0); // Maxwell scheduling word, absent from native instructions. } - code.push_back(instruction); + lowered_code.push_back(instruction); } // Keep the original padding ABI, then retain the native address map as // unreachable data. Existing disk-cache serialization carries it with code. - code.push_back(0); - Shader::Ampere::AppendNativeAddressMap(code, lowered.source_instruction_offsets); - cached_lowest = 0; - cached_highest = static_cast<u32>((code.size() - 1) * sizeof(u64)); + lowered_code.push_back(0); + Shader::Ampere::AppendNativeAddressMap(lowered_code, lowered.source_instruction_offsets); LOG_DEBUG(Shader, "Lowered native shader at {:#x}: {} source bytes, {} instructions", - program_base, native_source_size, lowered.instructions.size()); + program_base, result->source_size, lowered.instructions.size()); + return result; } GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, Tegra::MemoryManager& gpu_memory_, Maxwell::ShaderType program, GPUVAddr program_base_, - u32 start_address_) + u32 start_address_, + std::shared_ptr<const LoweredNativeCode> lowered_native_) : GenericEnvironment{gpu_memory_, program_base_, start_address_}, maxwell3d{&maxwell3d_} { gpu_memory->ReadBlock(program_base + start_address, &sph, sizeof(sph)); initial_offset = sizeof(sph); texture_bound = maxwell3d->regs.bindless_texture_const_buffer_slot; - if (maxwell3d->GetNativeMacroEngine().native_turing) { - if (program != Maxwell::ShaderType::VertexB && program != Maxwell::ShaderType::Pixel) { + const auto& regs{maxwell3d->regs}; + const bool native{maxwell3d->GetNativeMacroEngine().native_turing}; + // Task and mesh programs: the task in the vertex slot, its mesh program in the tessellation + // slot (without a task, the mesh program is in the vertex slot). + const bool mesh_slot{native && regs.IsMeshEnabled() && + (program == Maxwell::ShaderType::VertexB || + program == Maxwell::ShaderType::Tessellation)}; + if (native) { + if (program != Maxwell::ShaderType::VertexB && program != Maxwell::ShaderType::Pixel && + !mesh_slot) { throw Shader::RuntimeError("Unsupported native graphics shader stage {}", static_cast<u32>(program)); } if (std::getenv("EDEN_NATIVE_DRAW_DIAGNOSTICS")) { @@ -484,7 +570,7 @@ GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, dump.write(reinterpret_cast<const char*>(bytes.data()), bytes.size()); } } - InitializeNativeCode(); + InitializeNativeCode(std::move(lowered_native_)); } gp_passthrough_mask = maxwell3d->regs.post_vtg_shader_attrib_skip_mask; switch (program) { @@ -516,6 +602,19 @@ GraphicsEnvironment::GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, ASSERT_MSG(false, "Invalid program={}", program); break; } + if (mesh_slot) { + const bool has_task{regs.IsShaderConfigEnabled(Maxwell::ShaderType::Tessellation)}; + stage = program == Maxwell::ShaderType::VertexB && has_task ? Shader::Stage::Task + : Shader::Stage::Mesh; + mesh_registers = {regs.reg_array[Maxwell::MeshShaderA], regs.reg_array[Maxwell::MeshShaderB], + regs.reg_array[Maxwell::MeshInitShader]}; + if (stage == Shader::Stage::Mesh && regs.IsShaderConfigEnabled(Maxwell::ShaderType::Geometry)) { + const auto& geometry{regs.pipelines[static_cast<size_t>(Maxwell::ShaderType::Geometry)]}; + const GPUVAddr address{(static_cast<u64>(geometry.reserved[0] & 0xff) << 32) | + geometry.reserved[1]}; + gpu_memory->ReadBlock(address, &mesh_primitive_header, sizeof(mesh_primitive_header)); + } + } const u64 local_size{sph.LocalMemorySize()}; ASSERT(local_size <= (std::numeric_limits<u32>::max)()); local_memory_size = static_cast<u32>(local_size) + sph.common3.shader_local_memory_crs_size; @@ -594,7 +693,8 @@ u32 GraphicsEnvironment::ReadViewportTransformState() { ComputeEnvironment::ComputeEnvironment(Tegra::Engines::KeplerCompute& kepler_compute_, Tegra::MemoryManager& gpu_memory_, GPUVAddr program_base_, - u32 start_address_) + u32 start_address_, + std::shared_ptr<const LoweredNativeCode> lowered_native_) : GenericEnvironment{gpu_memory_, program_base_, start_address_}, kepler_compute{ &kepler_compute_} { const auto& qmd{kepler_compute->launch_description}; @@ -607,7 +707,7 @@ ComputeEnvironment::ComputeEnvironment(Tegra::Engines::KeplerCompute& kepler_com if (kepler_compute->IsNativeMode()) { initial_offset = sizeof(sph); is_proprietary_driver = true; - InitializeNativeCode(0); + InitializeNativeCode(std::move(lowered_native_), 0); start_address = sizeof(sph); } } @@ -712,7 +812,8 @@ void FileEnvironment::Deserialize(std::ifstream& file) { file.read(reinterpret_cast<char*>(&gp_passthrough_mask), sizeof(gp_passthrough_mask)); } } - is_proprietary_driver = texture_bound == 2; + // Native programs fold bank 1 reads as they did when they were built. + is_proprietary_driver = texture_bound == 2 || Shader::Ampere::HasNativeAddressMap(code); } void FileEnvironment::Dump(u64 pipeline_hash, u64 shader_hash) { diff --git a/src/video_core/shader_environment.h b/src/video_core/shader_environment.h index e49f01d95f..30d59955af 100644 --- a/src/video_core/shader_environment.h +++ b/src/video_core/shader_environment.h @@ -29,6 +29,19 @@ class Memorymanager; namespace VideoCommon { +/// A native (Switch 2) program after capture and lowering to Maxwell code. The shader cache keeps +/// it so the environments built for the program's pipelines don't capture and lower it again. +struct LoweredNativeCode { + /// Shader header, lowered instructions with Maxwell scheduling words, then the native + /// address map (what the environment's `code` used to be filled with). + std::vector<u64> code; + /// Bytes of the native program that were read, header included. + size_t source_size{}; + /// Constant buffer holding the texture handles if the program requires one, otherwise ~0U + /// (the engine's register value is used). + u32 texture_bound = ~0U; +}; + class GenericEnvironment : public Shader::Environment { public: explicit GenericEnvironment() = default; @@ -63,6 +76,10 @@ public: return native_source_size; } + [[nodiscard]] const std::shared_ptr<const LoweredNativeCode>& LoweredNative() const noexcept { + return lowered_native; + } + [[nodiscard]] bool CanBeSerialized() const noexcept; [[nodiscard]] u64 CalculateHash() const; @@ -78,8 +95,12 @@ public: protected: std::optional<u64> TryFindSize(); - void InitializeNativeCode(u32 source_header_size = 128); + /// Adopts an already lowered native program, or captures and lowers it when there is none. + void InitializeNativeCode(std::shared_ptr<const LoweredNativeCode> lowered, + u32 source_header_size = 128); + std::shared_ptr<const LoweredNativeCode> LowerNativeCode(u32 source_header_size); size_t native_source_size{}; + std::shared_ptr<const LoweredNativeCode> lowered_native; Tegra::Texture::TICEntry ReadTextureInfo(GPUVAddr tic_addr, u32 tic_limit, bool via_header_index, u32 raw); @@ -117,7 +138,8 @@ public: explicit GraphicsEnvironment(Tegra::Engines::Maxwell3D& maxwell3d_, Tegra::MemoryManager& gpu_memory_, Tegra::Engines::Maxwell3D::Regs::ShaderType program, - GPUVAddr program_base_, u32 start_address_); + GPUVAddr program_base_, u32 start_address_, + std::shared_ptr<const LoweredNativeCode> lowered_native_ = {}); ~GraphicsEnvironment() override = default; @@ -143,7 +165,8 @@ public: explicit ComputeEnvironment() = default; explicit ComputeEnvironment(Tegra::Engines::KeplerCompute& kepler_compute_, Tegra::MemoryManager& gpu_memory_, GPUVAddr program_base_, - u32 start_address_); + u32 start_address_, + std::shared_ptr<const LoweredNativeCode> lowered_native_ = {}); ~ComputeEnvironment() override = default; diff --git a/src/video_core/shader_handle_program.h b/src/video_core/shader_handle_program.h new file mode 100644 index 0000000000..3d35524cc8 --- /dev/null +++ b/src/video_core/shader_handle_program.h @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "common/common_types.h" +#include "shader_recompiler/shader_info.h" +#include "video_core/memory_manager.h" + +namespace VideoCommon { + +/// The texture handle a shader reads from a storage buffer, computed from the current constant +/// buffer and storage buffer words (Shader::Info::handle_programs). cbuf_address(index) is the +/// bound constant buffer's address, 0 when none is bound. +template <typename CbufAddress> +u32 EvaluateTextureHandle(const Shader::Info& info, u32 program, Tegra::MemoryManager& gpu_memory, + CbufAddress&& cbuf_address) { + return Shader::EvaluateHandleProgram( + info.handle_programs[program], + [&](u32 index, u32 offset) -> u32 { + const GPUVAddr base{cbuf_address(index)}; + return base != 0 ? gpu_memory.Read<u32>(base + offset) : 0U; + }, + [&](u32 binding, u32 offset) -> u32 { + if (binding >= info.storage_buffers_descriptors.size()) { + return 0U; + } + const auto& descriptor{info.storage_buffers_descriptors[binding]}; + const GPUVAddr cbuf{cbuf_address(descriptor.cbuf_index)}; + if (cbuf == 0) { + return 0U; + } + const GPUVAddr base{gpu_memory.Read<u64>(cbuf + descriptor.cbuf_offset)}; + return base != 0 ? gpu_memory.Read<u32>(base + offset) : 0U; + }) + .value_or(0U); +} + +} // namespace VideoCommon diff --git a/src/video_core/surface.cpp b/src/video_core/surface.cpp index 66b9577cb5..ef4859ea3e 100644 --- a/src/video_core/surface.cpp +++ b/src/video_core/surface.cpp @@ -214,6 +214,8 @@ PixelFormat PixelFormatFromGPUPixelFormat(Service::android::PixelFormat format) return PixelFormat::R5G6B5_UNORM; case Service::android::PixelFormat::Bgra8888: return PixelFormat::B8G8R8A8_UNORM; + case Service::android::PixelFormat::Rgba1010102: + return PixelFormat::A2B10G10R10_UNORM; default: UNIMPLEMENTED_MSG("Unimplemented format={}", format); return PixelFormat::A8B8G8R8_UNORM; diff --git a/src/video_core/texture_cache/descriptor_table.h b/src/video_core/texture_cache/descriptor_table.h index e40c128ab5..75383809c5 100644 --- a/src/video_core/texture_cache/descriptor_table.h +++ b/src/video_core/texture_cache/descriptor_table.h @@ -30,11 +30,17 @@ public: } void Invalidate() noexcept { - std::ranges::fill(read_descriptors, 0); + // Cleared on the next read: the texture cache invalidates every channel's tables for + // each image it deletes, dozens of times a frame while its collector runs. + invalidated = true; } [[nodiscard]] std::pair<T, bool> Read(Tegra::MemoryManager const& gpu_memory, u32 index) noexcept { DEBUG_ASSERT(index <= current_limit); + if (invalidated) [[unlikely]] { + std::ranges::fill(read_descriptors, 0); + invalidated = false; + } const GPUVAddr gpu_addr = current_gpu_addr + index * sizeof(T); std::pair<T, bool> result; gpu_memory.ReadBlockUnsafe(gpu_addr, std::addressof(result.first), sizeof(T)); @@ -60,6 +66,7 @@ public: read_descriptors.resize(Common::DivCeil(num_descriptors, 64U)); old_size = (std::min)(old_size, read_descriptors.size()); std::fill(read_descriptors.begin(), read_descriptors.begin() + old_size, 0); + invalidated = false; // descriptors.resize(num_descriptors); } @@ -68,6 +75,7 @@ public: std::vector<T> descriptors; GPUVAddr current_gpu_addr{}; u32 current_limit{}; + bool invalidated{}; ///< read_descriptors is to be cleared before the next read }; } // namespace VideoCommon diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 33ca051482..3f3ae9cafd 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -125,6 +125,7 @@ template <class P> void TextureCache<P>::RunGarbageCollector() { bool high_priority_mode = false; bool aggressive_mode = false; + bool evict_old_costly = false; u64 ticks_to_destroy = 0; size_t num_iterations = 0; const auto Configure = [&](bool allow_aggressive) { @@ -139,7 +140,8 @@ void TextureCache<P>::RunGarbageCollector() { ticks_to_destroy = (std::max)(ticks_to_destroy, min_age); num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10); }; - const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode](ImageId image_id) { + const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode, + &evict_old_costly](ImageId image_id) { if (num_iterations == 0) { return true; } @@ -149,7 +151,8 @@ void TextureCache<P>::RunGarbageCollector() { return false; } const bool must_download = IsDownloadable(image) && False(image.flags & ImageFlagBits::BadOverlap); - if ((!aggressive_mode && True(image.flags & ImageFlagBits::CostlyLoad)) || (!high_priority_mode && must_download)) { + if ((!aggressive_mode && !evict_old_costly && True(image.flags & ImageFlagBits::CostlyLoad)) || + (!high_priority_mode && must_download)) { return false; } if (must_download) { @@ -175,16 +178,84 @@ void TextureCache<P>::RunGarbageCollector() { } return false; }; + // Deleting by age alone left every device memory block partly used: the collector deletes + // thousands of images a minute and new ones land in the holes, so blocks seldom empty (title + // 1: ~0.8 GB of holes in ~190 32 MiB blocks after ten minutes, which the driver counts as + // used). Among the oldest candidates (8 per image this pass may delete), those in the + // emptiest blocks go first, so blocks drain and go back to the driver; placement already + // prefers the fullest blocks (best fit). Images with memory of their own sit in the middle. + const auto Collect = [&] { + if constexpr (requires(const Image& image) { image.MemoryBlock(); }) { + std::vector<ImageId> candidates; + const size_t limit = num_iterations * 8; + lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, [&](ImageId image_id) { + candidates.push_back(image_id); + return candidates.size() >= limit; + }); + if (candidates.empty()) { + return; + } + if (block_image_bytes_frame == ~0ULL || frame_tick - block_image_bytes_frame >= 60) { + block_image_bytes_frame = frame_tick; + block_image_bytes.clear(); + for (const auto [image_id, image] : slot_images) { + const auto block = image->MemoryBlock(); + if (block.block != 0) { + block_image_bytes[block.block] += block.size; + } + } + } + std::vector<std::pair<double, ImageId>> ordered; + ordered.reserve(candidates.size()); + for (const ImageId image_id : candidates) { + const auto block = slot_images[image_id].MemoryBlock(); + double fill = 0.5; + if (block.block != 0 && block.block_size != 0) { + const auto it = block_image_bytes.find(block.block); + fill = it != block_image_bytes.end() + ? static_cast<double>(it->second) / static_cast<double>(block.block_size) + : 0.0; + } + ordered.emplace_back(fill, image_id); + } + std::stable_sort(ordered.begin(), ordered.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); + for (const auto& [fill, image_id] : ordered) { + if (Cleanup(image_id)) { + break; + } + } + } else { + lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + } + }; Configure(false); - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + static const u64 old_age = [] { + const char* value = std::getenv("EDEN_NX2_TC_OLD_AGE"); + return value != nullptr ? std::strtoull(value, nullptr, 10) : 1800ULL; + }(); + if (high_priority_mode && old_age != 0 && frame_tick > old_age) { + evict_old_costly = true; + lru_cache.ForEachItemBelow(frame_tick - old_age, Cleanup); + evict_old_costly = false; + } + Collect(); if (total_used_memory >= critical_memory) { Configure(true); - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + Collect(); } } template <class P> void TextureCache<P>::ApplyThresholdOverrides(bool log) { + // On every GPU: past what a frame needs, more device memory only keeps the textures of earlier + // scenes (title 1 filled 6.7 GB of an 8 GB card and draws its races in ~5 GB), so collect from + // MaxCriticalMemory on rather than near the card's budget. + critical_memory = (std::min)(critical_memory, MaxCriticalMemory); + expected_memory = + (std::min)(expected_memory, critical_memory - (std::min)(critical_memory, u64{768_MiB})); + minimum_memory = + (std::min)(minimum_memory, expected_memory - (std::min)(expected_memory, u64{1_GiB})); // EDEN_NX2_TC_EXPECTED_MIB / EDEN_NX2_TC_CRITICAL_MIB override the collector's thresholds. for (const auto& [name, target] : {std::pair{"EDEN_NX2_TC_EXPECTED_MIB", &expected_memory}, std::pair{"EDEN_NX2_TC_CRITICAL_MIB", &critical_memory}}) { @@ -230,26 +301,78 @@ void TextureCache<P>::TickFrame() { // budget as the card fills (16 GB card: 13.5 GB left mid-race), hence the re-read every 30 // frames. EDEN_NX2_TC_*_MIB overrides still win. if constexpr (requires { runtime.GetDeviceMemoryBudget(); }) { + if constexpr (requires { runtime.MemoryPressureEvents(); }) { + // Images that did not fit (made in host memory or with nothing bound) mean the + // thresholds sit too high for this card and driver: leave more room until the + // pressure has been gone for a while. + if (native_thresholds_applied) { + const u32 events = runtime.MemoryPressureEvents(); + if (events != seen_pressure_events) { + seen_pressure_events = events; + pressure_margin = (std::min)(pressure_margin + 512_MiB, 3_GiB); + pressure_quiet_frames = 0; + budget_check_frames = 0; // thresholds are recomputed below + LOG_WARNING(HW_GPU, "Out of device memory: keeping {} MiB more free", + pressure_margin >> 20); + } else if (pressure_margin != 0 && ++pressure_quiet_frames % 600 == 0) { + pressure_margin -= (std::min)(pressure_margin, 128_MiB); + budget_check_frames = 0; + } + } + } if (native_thresholds_applied && budget_check_frames++ % 30 == 0) { const u64 budget = runtime.GetDeviceMemoryBudget(); if (budget > 2_GiB) { const u64 old_critical = critical_memory; const u64 margin = (std::min)(budget / 8, 1_GiB); - critical_memory = budget - margin; - expected_memory = budget - 2 * margin; + if constexpr (requires { runtime.GetDeviceMemorySlack(); }) { + // The usage counted here leaves out the unused part of the allocator's + // blocks, which the driver counts and a full heap cannot hand to a new + // image (RX 7800 XT: 1.06 GiB of it, the whole margin, with the heap at + // its budget for a minute before images stopped fitting). Reserve it + // too: at once when it grows, slowly when it shrinks, so evicting images + // (which leaves partly used blocks) does not chase the threshold down. + const u64 slack = (std::min)(runtime.GetDeviceMemorySlack(), 1536_MiB); + slack_margin = slack >= slack_margin + ? slack + : slack_margin - (std::min)(slack_margin - slack, 16_MiB); + } + critical_memory = + budget - (std::min)(margin + pressure_margin + slack_margin, budget / 2); + expected_memory = critical_memory - (std::min)(margin, critical_memory / 2); // Log the first values and real changes (the budget moves by a few MiB all // the time). const u64 change = critical_memory > old_critical ? critical_memory - old_critical : old_critical - critical_memory; - ApplyThresholdOverrides(budget_check_frames == 1 || change >= 256_MiB); + const bool log = budget_check_frames == 1 || change >= 256_MiB; + ApplyThresholdOverrides(log); + if (log) { + LOG_INFO(HW_GPU, + "Memory margins: base {} MiB, out of memory {} MiB, block slack {} MiB", + margin >> 20, pressure_margin >> 20, slack_margin >> 20); + } } } } } + if constexpr (requires { runtime.CurrentTick(); }) { + while (!gpu_sentenced_images.empty() && + runtime.IsFree(gpu_sentenced_images.front().tick)) { + gpu_sentenced_bytes -= gpu_sentenced_images.front().bytes; + gpu_sentenced_images.pop_front(); + } + } // If we can obtain the memory info, use it instead of the estimate. if (runtime.CanReportMemoryUsage()) { total_used_memory = runtime.GetDeviceMemoryUsage(); + // Images the collector deleted keep their memory for TICKS_TO_DESTROY frames: counting + // it kept the collector in aggressive mode, deleting 40+ images a frame for frames after + // it had freed enough (title 1: frames over 100 ms for seconds at a time). + for (const u64 bytes : sentenced_image_bytes) { + total_used_memory -= (std::min)(total_used_memory, bytes); + } + total_used_memory -= (std::min)(total_used_memory, gpu_sentenced_bytes); } // Every 600 frames: device memory in use against the collector's thresholds, and what the // images themselves account for (VRAM pressure makes the collector thrash). @@ -316,10 +439,18 @@ void TextureCache<P>::TickFrame() { RunGarbageCollector(); } sentenced_images.Tick(); + sentenced_image_index = (sentenced_image_index + 1) % TICKS_TO_DESTROY; + sentenced_image_bytes[sentenced_image_index] = 0; sentenced_framebuffers.Tick(); sentenced_image_view.Tick(); TickAsyncDecode(); TickAsyncUnswizzle(); + if (frame_tick % 600 == 0 && !uploaded_ranges.empty()) { + for (const auto& [address, size] : uploaded_ranges) { + device_memory.HintCold(address, size); + } + uploaded_ranges.clear(); + } runtime.TickFrame(); ++frame_tick; @@ -678,6 +809,7 @@ typename P::Framebuffer* TextureCache<P>::GetFramebuffer() { template <class P> ImageViewId TextureCache<P>::VisitImageView(u32 index, bool compute) { auto& table = compute ? channel_state->compute_image_table : channel_state->graphics_image_table; + index = Tegra::Texture::TextureHeaderIndex(index, table.current_limit); if (index > table.current_limit) { LOG_DEBUG(HW_GPU, "Invalid image view index={}", index); return NULL_IMAGE_VIEW_ID; @@ -689,8 +821,13 @@ ImageViewId TextureCache<P>::VisitImageView(u32 index, bool compute) { if (IsValidEntry(*gpu_memory, descriptor)) { // Is new (registered view) on the texture cache side? const auto [pair, is_new_tc] = channel_state->image_views.try_emplace(descriptor); - if (is_new_tc) + if (is_new_tc) { pair->second = CreateImageView(descriptor); + if (pair->second != NULL_IMAGE_VIEW_ID) { + image_view_descriptors[pair->second].emplace_back(current_channel_id, + descriptor); + } + } PrepareImageView(pair->second, false, false); channel_state->image_view_ids.insert_or_assign(map_index, pair->second); return pair->second; @@ -957,6 +1094,73 @@ bool TextureCache<P>::BlitImage(const Tegra::Engines::Fermi2D::Surface& dst, } template <class P> +bool TextureCache<P>::TemporalUpscale(const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& src, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, + std::array<f32, 2> jitter, std::array<f32, 2> motion_scale) { + if constexpr (FRAMEBUFFER_BLITS) { + return false; + } else { + const Tegra::Engines::Fermi2D::Config copy{ + .operation = Tegra::Engines::Fermi2D::Operation::SrcCopy, + .filter = Tegra::Engines::Fermi2D::Filter::Bilinear, + .must_accelerate = true, + .dst_x0 = 0, + .dst_y0 = 0, + .dst_x1 = static_cast<s32>(dst.width), + .dst_y1 = static_cast<s32>(dst.height), + .src_x0 = 0, + .src_y0 = 0, + .src_x1 = static_cast<s32>(src.width), + .src_y1 = static_cast<s32>(src.height), + }; + const auto result = GetBlitImages(dst, src, copy); + if (!result) { + return false; + } + const ImageInfo motion_info(motion_vectors); + has_deleted_images = false; + const ImageId motion_id = + FindOrInsertImage(motion_info, motion_vectors.Address(), RelaxedOptions{}); + if (has_deleted_images || !motion_id) { + return false; // the colour and output ids may be stale; the caller blits instead + } + const ImageId dst_id = result->dst_id; + const ImageId src_id = result->src_id; + PrepareImage(src_id, false, false); + PrepareImage(motion_id, false, false); + PrepareImage(dst_id, true, false); + for (const ImageId id : {src_id, motion_id, dst_id}) { + if (True(slot_images[id].flags & ImageFlagBits::Rescaled) || + slot_images[id].info.num_samples != 1) { + return false; + } + } + const auto view = [&](ImageId id, const Tegra::Engines::Fermi2D::Surface& surface, + PixelFormat format) -> std::optional<std::pair<FramebufferId, ImageViewId>> { + const std::optional base = slot_images[id].TryFindBase(surface.Address()); + if (!base) { + return std::nullopt; + } + const SubresourceRange range{.base = *base, .extent = {1, 1}}; + return RenderTargetFromImage(id, ImageViewInfo(ImageViewType::e2D, format, range)); + }; + const auto src_view = view(src_id, src, result->src_format); + const auto motion_view = view(motion_id, motion_vectors, motion_info.format); + const auto dst_view = view(dst_id, dst, result->dst_format); + if (!src_view || !motion_view || !dst_view) { + return false; + } + // After the views: creating them may reallocate the framebuffer slots. + Framebuffer* const dst_framebuffer = &slot_framebuffers[dst_view->first]; + return runtime.TemporalUpscale(dst_framebuffer, slot_image_views[dst_view->second], + slot_image_views[src_view->second], + slot_image_views[motion_view->second], jitter, + motion_scale); + } +} + +template <class P> std::pair<typename P::ImageView*, bool> TextureCache<P>::TryFindFramebufferImageView( const Tegra::FramebufferConfig& config, DAddr cpu_addr) { // TODO: Properly implement this @@ -1365,6 +1569,21 @@ void TextureCache<P>::RefreshContents(Image& image, ImageId image_id) { auto staging = runtime.UploadStagingBuffer(MapSizeBytes(image)); UploadImageContents(image, staging); runtime.InsertUploadMemoryBarrier(); + // The host GPU has its own copy now and the guest rarely reads this memory again: TickFrame + // marks it cold, so the OS pages it out before anything else when it needs RAM (title 1 + // streams several GB of textures through it). + constexpr u64 ColdHintMinBytes = 64_KiB; + if (image.guest_size_bytes >= ColdHintMinBytes && + False(image.flags & ImageFlagBits::GpuModified)) { + if (True(image.flags & ImageFlagBits::Sparse)) { + // Non-contiguous: each mapped piece (title 1 maps its own memory into these). + ForEachSparseSegment(image, [&](GPUVAddr, DAddr address, size_t size) { + uploaded_ranges.emplace_back(address, size); + }); + } else { + uploaded_ranges.emplace_back(image.cpu_addr, image.guest_size_bytes); + } + } } template <class P> @@ -1791,9 +2010,14 @@ void TextureCache<P>::InvalidateScale(Image& image) { for (size_t c : active_channel_ids) { auto& channel_info = channel_storage[c]; +#ifndef NDEBUG + // A walk over every entry (title 1: ~5,400 in 5 channels) for each deleted image: debug + // builds only. The tables are invalidated below, so each entry is written again before + // it is read. if constexpr (ENABLE_VALIDATION) for (auto& e : channel_info.image_view_ids) e.second = CORRUPT_ID; +#endif channel_info.graphics_image_table.Invalidate(); channel_info.compute_image_table.Invalidate(); } @@ -2911,6 +3135,8 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) { } // TickFrame sets the total from the driver, which can be under the sum of these estimates. total_used_memory -= (std::min)(total_used_memory, ImageMemoryBytes(slot_images[image_id])); + const u64 freed_bytes = (image.HasScaled() ? GetScaledImageSizeBytes(image) : 0) + + ImageMemoryBytes(slot_images[image_id]); const GPUVAddr gpu_addr = image.gpu_addr; const auto alloc_it = image_allocs_table.find(gpu_addr); if (alloc_it == image_allocs_table.end()) { @@ -2975,19 +3201,32 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) { if (retained) { LOG_INFO(Render_Vulkan, "Retained image for queued download: id={}", image_id.Value()); } else if (!immediate_delete) { - sentenced_images.Push(std::move(slot_images[image_id])); + if constexpr (requires { runtime.CurrentTick(); }) { + gpu_sentenced_images.push_back( + {runtime.CurrentTick(), freed_bytes, std::move(slot_images[image_id])}); + gpu_sentenced_bytes += freed_bytes; + } else { + sentenced_images.Push(std::move(slot_images[image_id])); + sentenced_image_bytes[sentenced_image_index] += freed_bytes; + } } slot_images.erase(image_id); alloc_images.erase(alloc_image_it); if (alloc_images.empty()) { image_allocs_table.erase(alloc_it); + slot_image_allocs.erase(alloc_id); } for (size_t c : active_channel_ids) { auto& channel_info = channel_storage[c]; +#ifndef NDEBUG + // A walk over every entry (title 1: ~5,400 in 5 channels) for each deleted image: debug + // builds only. The tables are invalidated below, so each entry is written again before + // it is read. if constexpr (ENABLE_VALIDATION) for (auto& e : channel_info.image_view_ids) e.second = CORRUPT_ID; +#endif channel_info.graphics_image_table.Invalidate(); channel_info.compute_image_table.Invalidate(); } @@ -2996,17 +3235,23 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) { template <class P> void TextureCache<P>::RemoveImageViewReferences(std::span<const ImageViewId> removed_views) { - for (size_t c : active_channel_ids) { - auto& channel_info = channel_storage[c]; - auto it = channel_info.image_views.begin(); - while (it != channel_info.image_views.end()) { - const auto found = std::ranges::find(removed_views, it->second); - if (found != removed_views.end()) { - it = channel_info.image_views.erase(it); - } else { - ++it; + for (const ImageViewId removed : removed_views) { + const auto refs = image_view_descriptors.find(removed); + if (refs == image_view_descriptors.end()) { + continue; + } + for (const auto& [channel, descriptor] : refs->second) { + // The channel may be gone, or its entry point to another view since. + if (std::ranges::find(active_channel_ids, channel) == active_channel_ids.end()) { + continue; + } + auto& image_views = channel_storage[channel].image_views; + if (const auto it = image_views.find(descriptor); + it != image_views.end() && it->second == removed) { + image_views.erase(it); } } + image_view_descriptors.erase(refs); } } @@ -3023,6 +3268,7 @@ void TextureCache<P>::RemoveFramebuffers(std::span<const ImageViewId> removed_vi last_framebuffer_serial = 0; } sentenced_framebuffers.Push(std::move(slot_framebuffers[framebuffer_id])); + slot_framebuffers.erase(framebuffer_id); it = framebuffers.erase(it); } else { ++it; diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 141e5b76d5..09deb8da02 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -136,6 +136,10 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI u64 native_critical_memory{}; u64 native_expected_memory{}; u64 budget_check_frames{}; + u32 seen_pressure_events{}; ///< Allocator pressure events already answered + u64 pressure_margin{}; ///< Extra room under the budget after images did not fit + u64 pressure_quiet_frames{}; ///< Frames since the last such image + u64 slack_margin{}; ///< Extra room for allocator block space the driver counts as used u64 astc_opaque_images{}; u64 astc_alpha_images{}; std::vector<u8> astc_scan_buffer; @@ -144,6 +148,9 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI static constexpr s64 DEFAULT_EXPECTED_MEMORY = 1_GiB + 125_MiB; static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB + 625_MiB; + /// The most device memory the collector lets the cache fill before it collects in earnest, + /// on every GPU (ApplyThresholdOverrides). + static constexpr u64 MaxCriticalMemory = 5_GiB; static constexpr size_t GC_EMERGENCY_COUNTS = 2; using Runtime = typename P::Runtime; @@ -276,6 +283,13 @@ public: const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Config& copy); + /// Stand-in for a DLSS evaluation: temporal upscale of src into dst (see the runtime's + /// TemporalUpscale). False when the images cannot take it; the caller then blits. + bool TemporalUpscale(const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& src, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, + std::array<f32, 2> jitter, std::array<f32, 2> motion_scale); + /// Try to find a cached image view in the given CPU address [[nodiscard]] std::pair<ImageView*, bool> TryFindFramebufferImageView( const Tegra::FramebufferConfig& config, DAddr cpu_addr); @@ -445,6 +459,12 @@ private: /// Remove image views references from the cache void RemoveImageViewReferences(std::span<const ImageViewId> removed_views); + /// Per image view, the (channel, descriptor) entries of the channels' image_views maps that + /// point to it. Deleting an image erased them by scanning every map (title 1: ~5,000 entries, + /// 50-130 us for each deleted image, dozens a frame while the collector runs). + std::unordered_map<ImageViewId, boost::container::small_vector<std::pair<size_t, TICEntry>, 2>> + image_view_descriptors; + /// Remove framebuffers using the given image views from the cache void RemoveFramebuffers(std::span<const ImageViewId> removed_views); @@ -564,6 +584,25 @@ private: static constexpr size_t TICKS_TO_DESTROY = 8; #endif DelayedDestructionRing<Image, TICKS_TO_DESTROY> sentenced_images; + /// Device memory of the images in each slot of sentenced_images (same index): deleted, but + /// counted in the driver's usage until the ring destroys them. + std::array<u64, TICKS_TO_DESTROY> sentenced_image_bytes{}; + size_t sentenced_image_index = 0; + /// Deleted images on runtimes that report GPU progress (CurrentTick, IsFree): destroyed, with + /// their device memory, as soon as the GPU is past them rather than TICKS_TO_DESTROY frames + /// later. While the collector deletes dozens a frame, eight frames of them held 1-1.5 GB of + /// VRAM above its threshold (title 1 on an 8 GB card: 6.6-7.0 GB in use against 5 GiB). + struct GpuSentencedImage { + u64 tick; + u64 bytes; + Image image; + }; + std::deque<GpuSentencedImage> gpu_sentenced_images; + u64 gpu_sentenced_bytes{}; + + /// Bytes of images per device memory block, for the collector (refreshed every 60 frames). + std::unordered_map<u64, u64> block_image_bytes; + u64 block_image_bytes_frame{~0ULL}; DelayedDestructionRing<ImageView, TICKS_TO_DESTROY> sentenced_image_view; DelayedDestructionRing<Framebuffer, TICKS_TO_DESTROY> sentenced_framebuffers; @@ -578,6 +617,8 @@ private: Common::ThreadWorker texture_decode_worker{1, "TextureDecoder", {}, Common::ThreadPlacement::Efficiency}; std::vector<std::unique_ptr<AsyncDecodeContext>> async_decodes; + /// Guest memory images were uploaded from since the last cold hint (device address, size). + std::vector<std::pair<DAddr, u64>> uploaded_ranges; std::deque<PendingUnswizzle> unswizzle_queue; u8 current_unswizzle_frame; diff --git a/src/video_core/textures/astc.cpp b/src/video_core/textures/astc.cpp index 950ce28d6b..a0d7e6e455 100644 --- a/src/video_core/textures/astc.cpp +++ b/src/video_core/textures/astc.cpp @@ -2038,38 +2038,74 @@ void Decompress(std::span<const uint8_t> data, uint32_t width, uint32_t height, } } +// What IsOpaque needs from a block's 11 mode bits, which are all DecodeBlockInfo reads (a void +// extent block also needs bits 10 and 11 set, checked per block). +struct BlockModeInfo { + bool error; + bool void_extent; + bool void_extent_hdr; + bool dual_plane; + u16 weight_bits; +}; + +static std::array<BlockModeInfo, 2048> MakeBlockModeInfos() { + std::array<BlockModeInfo, 2048> infos{}; + for (u32 mode = 0; mode < infos.size(); ++mode) { + // Bit 11 set, so a void extent mode decodes without its reserved-bit error. + const std::array<u8, 16> block{static_cast<u8>(mode), static_cast<u8>((mode >> 8) | 0x8)}; + InputBitStream strm(block); + const TexelWeightParams params = DecodeBlockInfo(strm); + BlockModeInfo& info = infos[mode]; + info.void_extent = params.m_bVoidExtentLDR || params.m_bVoidExtentHDR; + info.void_extent_hdr = params.m_bVoidExtentHDR; + info.error = params.m_bError && !info.void_extent; + info.dual_plane = params.m_bDualPlane; + info.weight_bits = info.error || info.void_extent + ? u16{0} + : static_cast<u16>(params.GetPackedBitSize()); + } + return infos; +} + bool IsOpaque(std::span<const uint8_t> data) { // Endpoint modes with an alpha channel: LDR luminance+alpha (4, 5), RGB scale + two alpha // (10), LDR RGBA (12, 13) and HDR RGBA (14, 15). constexpr u32 AlphaModes = (1U << 4) | (1U << 5) | (1U << 10) | (1U << 12) | (1U << 13) | (1U << 14) | (1U << 15); + // Decoding each block's mode bit by bit took ~12% of title 1's long frames, where every + // new ASTC texture is scanned (its title screen streams them in by the thousand). + static const std::array<BlockModeInfo, 2048> mode_infos = MakeBlockModeInfos(); for (size_t offset = 0; offset + 16 <= data.size(); offset += 16) { const std::span<const u8> block = data.subspan(offset, 16); - InputBitStream strm(block); - const TexelWeightParams params = DecodeBlockInfo(strm); - if (params.m_bError) { + u32 low; + std::memcpy(&low, block.data(), sizeof(low)); + const BlockModeInfo& info = mode_infos[low & 0x7FF]; + if (info.error) { continue; // decodes to the opaque error colour } - if (params.m_bVoidExtentLDR || params.m_bVoidExtentHDR) { + if (info.void_extent) { + if ((low & 0xC00) != 0xC00) { + continue; // reserved bits clear: the error colour too + } // One RGBA16 colour in bits 64-127: UNORM16, or FP16 for HDR. const u32 alpha = block[14] | (u32{block[15]} << 8); - if (alpha != (params.m_bVoidExtentHDR ? 0x3C00U : 0xFFFFU)) { + if (alpha != (info.void_extent_hdr ? 0x3C00U : 0xFFFFU)) { return false; } continue; } - const u32 partitions = strm.ReadBits<2>() + 1; - if (partitions == 4 && params.m_bDualPlane) { + const u32 partitions = ((low >> 11) & 3) + 1; + if (partitions == 4 && info.dual_plane) { continue; } if (partitions == 1) { - if ((AlphaModes >> strm.ReadBits<4>()) & 1) { + if ((AlphaModes >> ((low >> 13) & 0xF)) & 1) { return false; } continue; } - strm.ReadBits<10>(); // partition index - const u32 base_cem = strm.ReadBits<6>(); + // Bits 13-22 are the partition index. + const u32 base_cem = (low >> 23) & 0x3F; const u32 base_mode = base_cem & 3; if (base_mode == 0) { if ((AlphaModes >> (base_cem >> 2)) & 1) { @@ -2078,7 +2114,7 @@ bool IsOpaque(std::span<const uint8_t> data) { continue; } // The rest of the per-partition modes sit just below the weights (DecompressBlock). - const u32 weight_bits = params.GetPackedBitSize(); + const u32 weight_bits = info.weight_bits; if (weight_bits < 24 || weight_bits > 96) { continue; } diff --git a/src/video_core/textures/texture.h b/src/video_core/textures/texture.h index 9fc6a631bd..79396d8a68 100644 --- a/src/video_core/textures/texture.h +++ b/src/video_core/textures/texture.h @@ -165,6 +165,17 @@ static_assert(sizeof(TextureHandle) == 4, "TextureHandle has wrong size"); } } +/// Some Switch 2 bindless handles (Switch Sports' scenery, read from a uniform register) hold +/// the texture header's byte offset in the pool, index * 32, where the handle normally has the +/// index. Read it as an offset only when the plain index falls past the pool and the offset +/// lands inside it, so handles that already resolve are left alone. +[[nodiscard]] constexpr u32 TextureHeaderIndex(u32 index, u32 limit) { + if (index > limit && index % 32 == 0 && index / 32 <= limit) { + return index / 32; + } + return index; +} + struct TICEntry { union { struct { diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index b65c721bb8..01e13cb384 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -779,10 +779,14 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR .flags = flags, .physicalDevice = physical, .device = *logical, - // 64 MiB blocks on discrete GPUs too: with 256 MiB ones, Switch 2 titles' mix of + // 16 MiB blocks on discrete GPUs too: with 256 MiB ones, Switch 2 titles' mix of // texture sizes left ~38% of the image blocks empty (2 GB of an 8 GB card in title - // 1), which the texture cache counted as used and kept evicting textures for. - .preferredLargeHeapBlockSize = 64u * 1024u * 1024u, + // 1), which the texture cache counted as used and kept evicting textures for. With + // 64 MiB ones textures created and deleted by the thousand still left ~1.7 GB of holes + // after ten minutes, with 32 MiB ones ~0.5 GB: a block drains only once all of its + // images are gone, and fewer per block drain sooner. VMA gives allocations over half a + // block (8 MiB) their own memory, which goes back to the driver when they are freed. + .preferredLargeHeapBlockSize = 16u * 1024u * 1024u, .pAllocationCallbacks = nullptr, .pDeviceMemoryCallbacks = nullptr, .pHeapSizeLimit = heap_size_limit != 0 ? heap_size_limits.data() : nullptr, @@ -804,9 +808,37 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR Device::~Device() { SaveStaticPipelineCache(); ShutdownGPULogging(); + for (auto& pools : separate_pools) { + for (VmaPool pool : pools) { + if (pool != VK_NULL_HANDLE) { + vmaDestroyPool(allocator, pool); + } + } + } vmaDestroyAllocator(allocator); } +VmaPool Device::GetSeparatePool(SeparatePool kind, u32 memory_type) const { + std::scoped_lock lk{separate_pools_mutex}; + VmaPool& pool = separate_pools[static_cast<u32>(kind)][memory_type]; + if (pool == VK_NULL_HANDLE) { + const VmaPoolCreateInfo ci{ + .memoryTypeIndex = memory_type, + .flags = 0, + .blockSize = SeparatePoolBlockSize, + .minBlockCount = 0, + .maxBlockCount = 0, + .priority = 0.f, + .minAllocationAlignment = 0, + .pMemoryAllocateNext = nullptr, + }; + if (vmaCreatePool(allocator, &ci, &pool) != VK_SUCCESS) { + pool = VK_NULL_HANDLE; + } + } + return pool; +} + void Device::LoadStaticPipelineCache() { const auto create = [this](size_t size, const void* data) { static_pipeline_cache = logical.CreatePipelineCache({ @@ -1515,6 +1547,15 @@ void Device::RemoveUnsuitableExtensions() { VK_KHR_PIPELINE_EXECUTABLE_PROPERTIES_EXTENSION_NAME); } + // VK_EXT_mesh_shader: Switch 2 titles draw some geometry with task and mesh programs. Only + // the two stages are used; the multiview, shading rate and query features need more. + features.mesh_shader.multiviewMeshShader = VK_FALSE; + features.mesh_shader.primitiveFragmentShadingRateMeshShader = VK_FALSE; + features.mesh_shader.meshShaderQueries = VK_FALSE; + extensions.mesh_shader = features.mesh_shader.taskShader && features.mesh_shader.meshShader; + RemoveExtensionFeatureIfUnsuitable(extensions.mesh_shader, features.mesh_shader, + VK_EXT_MESH_SHADER_EXTENSION_NAME); + // VK_KHR_shader_quad_control extensions.shader_quad_control = features.shader_quad_control.shaderQuadControl; RemoveExtensionFeatureIfUnsuitable(extensions.shader_quad_control, features.shader_quad_control, @@ -1643,6 +1684,16 @@ u64 Device::GetDeviceMemoryUsage() const { return result; } +u64 Device::GetDeviceMemorySlack() const { + if (allocator == nullptr) { + return 0; + } + std::array<VmaBudget, VK_MAX_MEMORY_HEAPS> heap_budgets{}; + vmaGetHeapBudgets(allocator, heap_budgets.data()); + const VmaStatistics& statistics = heap_budgets[main_heap].statistics; + return statistics.blockBytes - (std::min)(statistics.blockBytes, statistics.allocationBytes); +} + u64 Device::GetDeviceMemoryBudget() const { if (!extensions.memory_budget) { return 0; diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index 8237868c6d..622d792210 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -6,8 +6,10 @@ #pragma once +#include <array> #include <atomic> #include <cstdlib> +#include <mutex> #include <optional> #include <set> #include <span> @@ -21,6 +23,7 @@ #include "video_core/vulkan_common/vulkan_wrapper.h" VK_DEFINE_HANDLE(VmaAllocator) +VK_DEFINE_HANDLE(VmaPool) // Define all features which may be used by the implementation here. // Vulkan version in the macro describes the minimum version required for feature availability. @@ -66,6 +69,7 @@ VK_DEFINE_HANDLE(VmaAllocator) FEATURE(EXT, 4444Formats, 4444_FORMATS, format_a4b4g4r4) \ FEATURE(EXT, IndexTypeUint8, INDEX_TYPE_UINT8, index_type_uint8) \ FEATURE(EXT, LineRasterization, LINE_RASTERIZATION, line_rasterization) \ + FEATURE(EXT, MeshShader, MESH_SHADER, mesh_shader) \ FEATURE(EXT, PrimitiveTopologyListRestart, PRIMITIVE_TOPOLOGY_LIST_RESTART, \ primitive_topology_list_restart) \ FEATURE(EXT, ProvokingVertex, PROVOKING_VERTEX, provoking_vertex) \ @@ -264,6 +268,17 @@ public: return allocator; } + /// Allocations kept out of the default blocks, which then hold images only: churned buffers + /// and long-lived sparse pages left holes the texture cache's collector couldn't drain. + enum class SeparatePool : u32 { + SparsePages, + DeviceBuffers, + }; + static constexpr VkDeviceSize SeparatePoolBlockSize = 16ULL << 20; + + /// The pool of that kind for a memory type, created on first use (null if that failed). + VmaPool GetSeparatePool(SeparatePool kind, u32 memory_type) const; + /// Returns the logical device. const vk::Device& GetLogical() const { return logical; @@ -633,6 +648,12 @@ FN_MAX_LIMIT_LIST return extensions.shader_quad_control && features.shader_quad_control.shaderQuadControl; } + /// Returns true if the device supports task and mesh shaders (VK_EXT_mesh_shader). + bool IsExtMeshShaderSupported() const { + return extensions.mesh_shader && features.mesh_shader.taskShader && + features.mesh_shader.meshShader; + } + /// Returns true if the device supports VK_KHR_image_format_list. bool IsKhrImageFormatListSupported() const { return extensions.image_format_list || instance_version >= VK_API_VERSION_1_2; @@ -951,6 +972,11 @@ FN_MAX_LIMIT_LIST return 0x00010300U; } + /// Returns true for an integrated GPU (its device-local memory is system memory). + bool IsIntegrated() const { + return is_integrated; + } + /// Returns true when a known debugging tool is attached. bool HasDebuggingToolAttached() const { return has_renderdoc || has_nsight_graphics || has_radeon_gpu_profiler; @@ -993,9 +1019,15 @@ FN_MAX_LIMIT_LIST /// The driver's current memory budget for this process (VK_EXT_memory_budget), in /// GetDeviceMemoryUsage() terms: the usage at which the main device-local heap reaches its - /// budget. 0 without the extension. + /// budget. 0 without the extension. The unused part of the allocator's blocks is not + /// subtracted: see GetDeviceMemorySlack(). u64 GetDeviceMemoryBudget() const; + /// Memory the allocator holds from the driver in the main device-local heap but has no + /// allocation in (the unused part of its blocks). GetDeviceMemoryUsage() does not count it, + /// yet the driver does: with the heap at its budget it is not available to new images. + u64 GetDeviceMemorySlack() const; + u32 GetSetsPerPool() const { return sets_per_pool; } @@ -1179,6 +1211,8 @@ private: private: VkInstance instance; ///< Vulkan instance. VmaAllocator allocator{}; ///< VMA allocator. + mutable std::mutex separate_pools_mutex; + mutable std::array<std::array<VmaPool, VK_MAX_MEMORY_TYPES>, 2> separate_pools{}; vk::DeviceDispatch dld; ///< Device function pointers. vk::PhysicalDevice physical; ///< Physical device. vk::Device logical; ///< Logical device. diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index 1798611e26..a0a39abe39 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -209,7 +209,10 @@ MemoryAllocator::~MemoryAllocator() = default; vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicated) const { const VmaAllocationCreateInfo alloc_ci = { - .flags = static_cast<VmaAllocationCreateFlags>(VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT) | + // Best fit: images freed by the thousand leave holes in the blocks, which new ones + // should fill before VMA takes another block. + .flags = static_cast<VmaAllocationCreateFlags>(VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | + VMA_ALLOCATION_CREATE_STRATEGY_MIN_MEMORY_BIT) | (dedicated ? VmaAllocationCreateFlags{VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT} : VmaAllocationCreateFlags{0}), .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, @@ -242,11 +245,32 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicat ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels, ci.arrayLayers, static_cast<u32>(ci.format)); } + pressure_events.fetch_add(1, std::memory_order_relaxed); VmaAllocationCreateInfo relaxed = alloc_ci; relaxed.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; result = vmaCreateImage(allocator, &ci, &relaxed, &handle, &allocation, &alloc_info); } + if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && !huge) { + // Still out of device memory: a real image in host memory (slow, but valid) beats one + // with nothing bound, which AMD's drivers do not survive being sampled. + VmaAllocationCreateInfo host = alloc_ci; + host.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; + host.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST; + host.preferredFlags = 0; + result = vmaCreateImage(allocator, &ci, &host, &handle, &allocation, &alloc_info); + static u32 host_logged{}; + if (result == VK_SUCCESS && host_logged++ < 16) { + LOG_WARNING(Render_Vulkan, + "Image {}x{}x{} ({} levels, {} layers, format {}) is out of device memory; " + "placed in host memory", + ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels, + ci.arrayLayers, static_cast<u32>(ci.format)); + } + } if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && ci.imageType != VK_IMAGE_TYPE_1D) { + if (!huge) { + pressure_events.fetch_add(1, std::memory_order_relaxed); + } // Too big to back at all, e.g. title 1's sparse virtual texture arrays (8192x8192, // 68 layers, ~23 GB): on the console only resident tiles have memory. Use an image with // nothing bound, so views and copies still work, instead of aborting. @@ -313,6 +337,24 @@ VmaAllocation MemoryAllocator::AllocateSparseMemory(const VkMemoryRequirements & }; VmaAllocation allocation{}; VmaAllocationInfo info{}; + // In a pool of their own (see Device::SeparatePool); anything over half a block, or a pool + // allocation that fails, takes the default blocks. + u32 memory_type{}; + if (requirements.size <= Device::SeparatePoolBlockSize / 2 && + vmaFindMemoryTypeIndex(allocator, requirements.memoryTypeBits, &alloc_ci, &memory_type) == + VK_SUCCESS) { + if (const VmaPool pool = + device.GetSeparatePool(Device::SeparatePool::SparsePages, memory_type)) { + VmaAllocationCreateInfo pool_ci = alloc_ci; + pool_ci.pool = pool; + if (vmaAllocateMemory(allocator, &requirements, &pool_ci, &allocation, &info) == + VK_SUCCESS) { + memory = info.deviceMemory; + offset = info.offset; + return allocation; + } + } + } if (vmaAllocateMemory(allocator, &requirements, &alloc_ci, &allocation, &info) != VK_SUCCESS) { return VK_NULL_HANDLE; } @@ -327,6 +369,39 @@ void MemoryAllocator::FreeSparseMemory(VmaAllocation allocation) const { } } +bool MemoryAllocator::CreateBufferInSeparatePool(const VkBufferCreateInfo& ci, MemoryUsage usage, + const VmaAllocationCreateInfo& alloc_ci, + VkBuffer& handle, VmaAllocation& allocation, + VmaAllocationInfo& alloc_info) const { + // The buffer cache's buffers, churned by its collector: in a pool of their own (see + // Device::SeparatePool). Anything over half a block takes the default blocks (dedicated). + if (usage != MemoryUsage::DeviceLocal || ci.size > Device::SeparatePoolBlockSize / 2) { + return false; + } + u32 memory_type{}; + { + std::scoped_lock lk{buffer_type_mutex}; + const auto key = (u64{ci.usage} << 32) | ci.flags; + if (const auto it = buffer_memory_types.find(key); it != buffer_memory_types.end()) { + memory_type = it->second; + } else if (vmaFindMemoryTypeIndexForBufferInfo(allocator, &ci, &alloc_ci, &memory_type) == + VK_SUCCESS) { + buffer_memory_types.emplace(key, memory_type); + } else { + return false; + } + } + const VmaPool pool = device.GetSeparatePool(Device::SeparatePool::DeviceBuffers, memory_type); + if (pool == VK_NULL_HANDLE) { + return false; + } + VmaAllocationCreateInfo pool_ci = alloc_ci; + pool_ci.pool = pool; + pool_ci.memoryTypeBits = 0; + return vmaCreateBuffer(allocator, &ci, &pool_ci, &handle, &allocation, &alloc_info) == + VK_SUCCESS; +} + vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const { // MESA will do memcpy() if not marked as host cached, so just force mark it for most buffers auto const anv_flags = (usage == MemoryUsage::Stream @@ -348,7 +423,9 @@ vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsa VmaAllocation allocation{}; VkMemoryPropertyFlags property_flags{}; - vk::Check(vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + if (!CreateBufferInSeparatePool(ci, usage, alloc_ci, handle, allocation, alloc_info)) { + vk::Check(vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + } vmaGetAllocationMemoryProperties(allocator, allocation, &property_flags); // Log GPU memory allocation for buffers diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index e1c267ab79..8b9e72b3e3 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -6,8 +6,11 @@ #pragma once +#include <atomic> #include <memory> +#include <mutex> #include <span> +#include <unordered_map> #include <vector> #include "common/common_types.h" @@ -113,11 +116,23 @@ namespace Vulkan { vk::Image CreateUnbackedImage(const VkImageCreateInfo &ci, bool aliased = false) const; /// Device memory to bind to part of a sparse resource. Null when there is none left. + /// Device-local buffers in their own VMA pool; false when it doesn't apply or failed. + bool CreateBufferInSeparatePool(const VkBufferCreateInfo& ci, MemoryUsage usage, + const VmaAllocationCreateInfo& alloc_ci, VkBuffer& handle, + VmaAllocation& allocation, + VmaAllocationInfo& alloc_info) const; + VmaAllocation AllocateSparseMemory(const VkMemoryRequirements &requirements, VkDeviceMemory &memory, VkDeviceSize &offset) const; void FreeSparseMemory(VmaAllocation allocation) const; + /// Counts the images that did not fit the device memory budget (made anyway, in host + /// memory or with nothing bound): the texture cache collects harder when it grows. + u32 MemoryPressureEvents() const noexcept { + return pressure_events.load(std::memory_order_relaxed); + } + vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const; vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage, @@ -153,6 +168,10 @@ namespace Vulkan { const VkPhysicalDeviceMemoryProperties properties; ///< Physical device memory properties. VkDeviceSize buffer_image_granularity; ///< Adjacent buffer/image granularity u32 valid_memory_types{~0u}; + mutable std::atomic<u32> pressure_events{0}; + mutable std::mutex buffer_type_mutex; + /// Memory type of device-local buffers by usage and flags (CreateBufferInSeparatePool). + mutable std::unordered_map<u64, u32> buffer_memory_types; }; /// Logs VMA's blocks and allocations per memory type, and each heap's budget. diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index dbaaf450a0..d0bca88125 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -116,6 +116,7 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkCmdDrawIndirectCount); X(vkCmdDrawIndexedIndirectCount); X(vkCmdDrawIndirectByteCountEXT); + X(vkCmdDrawMeshTasksEXT); X(vkCmdEndConditionalRenderingEXT); X(vkCmdEndQuery); X(vkCmdEndRenderPass); @@ -525,6 +526,19 @@ void Image::Release() const noexcept { } } +Image::MemoryBlock Image::GetMemoryBlock() const noexcept { + if (allocation == nullptr) { + return {}; + } + VmaAllocationInfo2 info{}; + vmaGetAllocationInfo2(allocator, allocation, &info); + if (info.dedicatedMemory) { + return {0, info.blockSize, info.allocationInfo.size}; + } + return {reinterpret_cast<u64>(info.allocationInfo.deviceMemory), info.blockSize, + info.allocationInfo.size}; +} + void Buffer::Flush() const { if (!is_coherent) { vmaFlushAllocation(allocator, allocation, 0, VK_WHOLE_SIZE); diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index 3010368973..f49e9518bb 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -232,6 +232,7 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkCmdDrawIndirectCount vkCmdDrawIndirectCount{}; PFN_vkCmdDrawIndexedIndirectCount vkCmdDrawIndexedIndirectCount{}; PFN_vkCmdDrawIndirectByteCountEXT vkCmdDrawIndirectByteCountEXT{}; + PFN_vkCmdDrawMeshTasksEXT vkCmdDrawMeshTasksEXT{}; PFN_vkCmdEndConditionalRenderingEXT vkCmdEndConditionalRenderingEXT{}; PFN_vkCmdEndDebugUtilsLabelEXT vkCmdEndDebugUtilsLabelEXT{}; PFN_vkCmdEndQuery vkCmdEndQuery{}; @@ -736,6 +737,15 @@ public: return usage; } + /// Where the image's memory lies: the block (0 when the image has no memory or a block of + /// its own), the block's size and the image's part of it. + struct MemoryBlock { + u64 block; + u64 block_size; + u64 size; + }; + [[nodiscard]] MemoryBlock GetMemoryBlock() const noexcept; + private: void Release() const noexcept; @@ -1319,6 +1329,10 @@ public: dld->vkCmdDrawIndexedIndirect(handle, src_buffer, src_offset, draw_count, stride); } + void DrawMeshTasksEXT(u32 group_count_x, u32 group_count_y, u32 group_count_z) const noexcept { + dld->vkCmdDrawMeshTasksEXT(handle, group_count_x, group_count_y, group_count_z); + } + void DrawIndirectCount(VkBuffer src_buffer, VkDeviceSize src_offset, VkBuffer count_buffer, VkDeviceSize count_offset, u32 draw_count, u32 stride) const noexcept { dld->vkCmdDrawIndirectCount(handle, src_buffer, src_offset, count_buffer, count_offset, diff --git a/src/yuzu/configuration/configure_graphics.cpp b/src/yuzu/configuration/configure_graphics.cpp index e4a140fff5..074b6f405b 100644 --- a/src/yuzu/configuration/configure_graphics.cpp +++ b/src/yuzu/configuration/configure_graphics.cpp @@ -218,6 +218,7 @@ void ConfigureGraphics::Setup(const ConfigurationShared::Builder& builder) { std::map<u32, QWidget*> hold_graphics; std::vector<QWidget*> hold_api; + QCheckBox* frame_gen_checkbox = nullptr; for (const auto setting : Settings::values.linkage.by_category[Settings::Category::Renderer]) { #ifndef HAS_RESHADE @@ -225,9 +226,6 @@ void ConfigureGraphics::Setup(const ConfigurationShared::Builder& builder) { continue; } #endif - if (setting->Id() == Settings::values.frame_gen.Id()) { - continue; - } ConfigurationShared::Widget* widget = [&]() { if (setting->Id() == Settings::values.fsr_sharpening_slider.Id()) { @@ -325,6 +323,9 @@ void ConfigureGraphics::Setup(const ConfigurationShared::Builder& builder) { hold_graphics.emplace(setting->Id(), widget); #endif + } else if (setting->Id() == Settings::values.frame_gen.Id()) { + frame_gen_checkbox = widget->checkbox; + hold_graphics.emplace(setting->Id(), widget); } else { hold_graphics.emplace(setting->Id(), widget); } @@ -334,6 +335,21 @@ void ConfigureGraphics::Setup(const ConfigurationShared::Builder& builder) { graphics_layout.addWidget(widget); } + // Hide rather than disable, a disabled widget would read as using the global value + if (frame_gen_checkbox != nullptr) { + for (const u32 id : {Settings::values.frame_gen_type.Id(), + Settings::values.frame_gen_multiplier.Id(), + Settings::values.frame_gen_target_rate.Id(), + Settings::values.frame_gen_flow_scale_auto.Id(), + Settings::values.frame_gen_flow_scale.Id(), + Settings::values.frame_gen_queue_target.Id()}) { + if (const auto it = hold_graphics.find(id); it != hold_graphics.end()) { + it->second->setVisible(frame_gen_checkbox->isChecked()); + connect(frame_gen_checkbox, &QCheckBox::toggled, it->second, &QWidget::setVisible); + } + } + } + for (auto widget : hold_api) { api_grid_layout->addWidget(widget); } diff --git a/src/yuzu/configuration/shared_widget.cpp b/src/yuzu/configuration/shared_widget.cpp index 0e829e9100..4971c0cfa7 100644 --- a/src/yuzu/configuration/shared_widget.cpp +++ b/src/yuzu/configuration/shared_widget.cpp @@ -515,7 +515,13 @@ void Widget::SetupComponent(const QString& label, std::function<void()>& load_fu layout->setContentsMargins(0, 0, 0, 0); if (other_setting == nullptr) { - other_setting = setting.PairedSetting(); + // A paired setting without the Paired specialization gets a widget of its own, and + // several settings can share it, so they must not each save their own checkbox to it + auto* const paired_setting = setting.PairedSetting(); + if (paired_setting != nullptr && + paired_setting->Specialization() == Settings::Specialization::Paired) { + other_setting = paired_setting; + } } const bool require_checkbox = other_setting != nullptr && other_setting->TypeId() == "bool"; diff --git a/src/yuzu/game/game_list.cpp b/src/yuzu/game/game_list.cpp index 17183c02ec..d2613697bf 100644 --- a/src/yuzu/game/game_list.cpp +++ b/src/yuzu/game/game_list.cpp @@ -24,6 +24,7 @@ #include "common/common_types.h" #include "core/core.h" +#include "core/file_sys/nup_writer.h" #include "core/file_sys/patch_manager.h" #include "core/file_sys/registered_cache.h" #include "qt_common/config/uisettings.h" @@ -99,6 +100,10 @@ GameList::GameList(FileSys::VirtualFilesystem vfs_, FileSys::ManualContentProvid ResetViewMode(); } +void GameList::SetNupConversionRunning(bool running) { + nup_conversion_running = running; +} + GameList::~GameList() { UnloadController(); } @@ -481,6 +486,11 @@ void GameList::AddGamePopup(QMenu& context_menu, u64 program_id, const std::stri QAction* dump_romfs_sdmc = dump_romfs_menu->addAction(tr("Dump RomFS to SDMC")); QAction* verify_integrity = context_menu.addAction(tr("Verify Integrity")); QAction* copy_tid = context_menu.addAction(tr("Copy Title ID to Clipboard")); + QAction* convert_nup = nullptr; + if (FileSys::Nup::DumpRootFromMain(path)) { + convert_nup = context_menu.addAction(tr("Convert to NUP...")); + convert_nup->setEnabled(!nup_conversion_running); + } #if !defined(__APPLE__) QMenu* shortcut_menu = context_menu.addMenu(tr("Create Shortcut")); QAction* create_desktop_shortcut = shortcut_menu->addAction(tr("Add to Desktop")); @@ -561,6 +571,10 @@ void GameList::AddGamePopup(QMenu& context_menu, u64 program_id, const std::stri [this, path]() { emit VerifyIntegrityRequested(path); }); connect(copy_tid, &QAction::triggered, this, [this, program_id]() { emit CopyTIDRequested(program_id); }); + if (convert_nup) { + connect(convert_nup, &QAction::triggered, this, + [this, path]() { emit ConvertToNupRequested(path); }); + } #if !defined(__APPLE__) connect(create_desktop_shortcut, &QAction::triggered, this, [this, program_id, path]() { emit CreateShortcut(program_id, path, QtCommon::Game::ShortcutTarget::Desktop); diff --git a/src/yuzu/game/game_list.h b/src/yuzu/game/game_list.h index 161e66195f..b33dfdd9c8 100644 --- a/src/yuzu/game/game_list.h +++ b/src/yuzu/game/game_list.h @@ -89,6 +89,7 @@ public: void ResetViewMode(); public slots: + void SetNupConversionRunning(bool running); void RefreshGameDirectory(); void RefreshExternalContent(); void UpdateIconSizes(); @@ -107,6 +108,7 @@ signals: void SetPlayTimeRequested(u64 program_id); void DumpRomFSRequested(u64 program_id, const std::string& game_path, DumpRomFSTarget target); void VerifyIntegrityRequested(const std::string& game_path); + void ConvertToNupRequested(const std::string& game_path); void CopyTIDRequested(u64 program_id); void CreateShortcut(u64 program_id, const std::string& game_path, const QtCommon::Game::ShortcutTarget target); @@ -166,6 +168,7 @@ private: Core::System& system; bool m_isTreeMode = true; + bool nup_conversion_running{}; QAbstractItemView* m_currentView = nullptr; }; diff --git a/src/yuzu/main_window.cpp b/src/yuzu/main_window.cpp index e374c9f27c..70000a22c9 100644 --- a/src/yuzu/main_window.cpp +++ b/src/yuzu/main_window.cpp @@ -152,6 +152,7 @@ static FileSys::VirtualFile VfsDirectoryCreateFileWrapper(const FileSys::Virtual #include "core/file_sys/card_image.h" #include "core/file_sys/common_funcs.h" +#include "core/file_sys/nup_writer.h" #include "core/file_sys/romfs.h" #include "core/file_sys/savedata_factory.h" @@ -453,7 +454,7 @@ MainWindow::MainWindow(bool has_broken_vulkan) QtCommon::SetupContentProviders(); // Gen keys if necessary - OnCheckFirmwareDecryption(); + OnCheckFirmwareDecryption(false); #ifdef __unix__ OnCheckGraphicsBackend(); @@ -557,6 +558,10 @@ MainWindow::MainWindow(bool has_broken_vulkan) } MainWindow::~MainWindow() { + if (nup_cancel) { + nup_cancel->store(true); + } + nup_watcher.waitForFinished(); #ifdef __APPLE__ AllowOSSleep(); #endif @@ -1460,6 +1465,8 @@ void MainWindow::ConnectWidgetEvents() { connect(game_list, &GameList::DumpRomFSRequested, this, &MainWindow::OnGameListDumpRomFS); connect(game_list, &GameList::VerifyIntegrityRequested, this, &MainWindow::OnGameListVerifyIntegrity); + connect(game_list, &GameList::ConvertToNupRequested, this, + &MainWindow::OnGameListConvertToNup); connect(game_list, &GameList::CopyTIDRequested, this, &MainWindow::OnGameListCopyTID); connect(game_list, &GameList::CreateShortcut, this, &MainWindow::OnGameListCreateShortcut); connect(game_list, &GameList::AddDirectory, this, &MainWindow::OnGameListAddDirectory); @@ -1891,6 +1898,12 @@ bool MainWindow::SelectAndSetCurrentUser( void MainWindow::BootGame(const QString& filename, Service::AM::FrontendAppletParameters params, StartGameType type) { + if (nup_running && filename.toStdString() == nup_source_path) { + QMessageBox::information(this, tr("Conversion in Progress"), + tr("This game cannot be started while its dump is being " + "converted.")); + return; + } if (boot_in_progress || close_pending) { return; } @@ -4030,6 +4043,151 @@ void MainWindow::OnGameListRefresh() { SetFirmwareVersion(); } +void MainWindow::OnGameListConvertToNup(const std::string& game_path) { + if (nup_running) + return; + + const auto root = FileSys::Nup::DumpRootFromMain(game_path); + if (!root) + return; + const QString convert_title = tr("Convert to NUP"); + const auto archive = FileSys::Nup::DefaultArchivePath(*root); + if (std::filesystem::exists(archive)) { + QMessageBox::information(this, convert_title, + tr("The target archive already exists:\n%1") + .arg(QString::fromStdString(Common::FS::PathToUTF8String(archive)))); + return; + } + + const bool resume = FileSys::Nup::HasResumableJournal(archive); + bool delete_sources = resume; + if (resume) { + if (QMessageBox::question(this, tr("Resume Conversion"), + tr("Resume the interrupted conversion?"), + QMessageBox::Yes | QMessageBox::No) != QMessageBox::Yes) + return; + } else { + QMessageBox prompt(QMessageBox::Question, convert_title, + tr("Source folder:\n%1\n\nTarget archive:\n%2") + .arg(QString::fromStdString(Common::FS::PathToUTF8String(*root)), + QString::fromStdString(Common::FS::PathToUTF8String(archive))), + QMessageBox::Yes | QMessageBox::No, this); + // QMessageBox takes ownership of its check box. + auto* low_storage = new QCheckBox(tr("Low storage: delete the original files as they " + "are packed (cannot be undone)")); + prompt.setCheckBox(low_storage); + u64 needed{}; + { + std::error_code ec; + for (std::filesystem::recursive_directory_iterator it(*root, ec), end; + !ec && it != end; it.increment(ec)) { + if (it->is_regular_file(ec)) + needed += it->file_size(ec); + } + } + std::error_code space_error; + const auto space = std::filesystem::space(archive.parent_path(), space_error); + const auto to_mb = [](u64 bytes) { return bytes / (1024 * 1024); }; + const QString space_info = tr("\n\nApproximate space needed: %1 MB\nFree space on target " + "volume: %2 MB") + .arg(to_mb(needed)) + .arg(space_error ? 0 : to_mb(space.available)); + prompt.setInformativeText(space_info); + if (!space_error && space.available < needed) { + prompt.setText(prompt.text() + tr("\n\nWarning: free space is less than the " + "estimated space needed.")); + } + if (prompt.exec() != QMessageBox::Yes) + return; + delete_sources = low_storage->isChecked(); + } + + nup_running = true; + nup_source_path = game_path; + game_list->SetNupConversionRunning(true); + nup_cancel = std::make_shared<std::atomic_bool>(false); + nup_progress = + new QProgressDialog(tr("Preparing conversion..."), tr("Cancel"), 0, 1000, this); + nup_progress->setWindowTitle(tr("Converting to NUP")); + nup_progress->setMinimumDuration(0); + nup_progress->setAutoClose(false); + nup_progress->setAutoReset(false); + connect(nup_progress, &QProgressDialog::canceled, this, + [cancel = nup_cancel] { cancel->store(true); }); + nup_progress->show(); + + const auto cancel = nup_cancel; + auto last_update = std::make_shared<std::chrono::steady_clock::time_point>(); + nup_watcher.setFuture(QtConcurrent::run([this, root = *root, archive, delete_sources, + cancel, last_update] { + std::string error; + const auto on_progress = [this, last_update]( + const FileSys::Nup::PackProgress& progress) { + const auto now = std::chrono::steady_clock::now(); + if (now - *last_update < std::chrono::milliseconds(200)) + return; + *last_update = now; + QMetaObject::invokeMethod( + this, + [this, progress] { + if (!nup_progress) + return; + const auto percent = progress.bytes_total == 0 + ? 0 + : static_cast<int>(progress.bytes_done * 1000 / + progress.bytes_total); + nup_progress->setValue(percent); + nup_progress->setLabelText( + tr("%1 / %2 MB\n%3 / %4 files") + .arg(progress.bytes_done / (1024 * 1024)) + .arg(progress.bytes_total / (1024 * 1024)) + .arg(progress.files_done) + .arg(progress.files_total)); + }, + Qt::QueuedConnection); + }; + const auto result = FileSys::Nup::PackDump( + root, archive, {.delete_sources = delete_sources}, on_progress, cancel.get(), &error); + QMetaObject::invokeMethod( + this, + [this, result, delete_sources, error] { + nup_running = false; + nup_source_path.clear(); + game_list->SetNupConversionRunning(false); + if (nup_progress) { + nup_progress->close(); + nup_progress->deleteLater(); + nup_progress = nullptr; + } + nup_cancel.reset(); + switch (result) { + case FileSys::Nup::PackResult::Success: + QMessageBox::information( + this, tr("Conversion Complete"), + delete_sources ? tr("The NUP was created and the loose folder was " + "removed.") + : tr("The NUP was created. You can delete the loose " + "folder if you no longer need it.")); + OnGameListRefresh(); + break; + case FileSys::Nup::PackResult::Cancelled: + if (delete_sources) + QMessageBox::information(this, tr("Conversion Cancelled"), + tr("The conversion can be resumed by choosing " + "Convert to NUP again.")); + break; + case FileSys::Nup::PackResult::InvalidSource: + case FileSys::Nup::PackResult::IoError: + QMessageBox::critical(this, tr("Conversion Failed"), + QString::fromStdString(error)); + break; + } + }, + Qt::QueuedConnection); + return result; + })); +} + void MainWindow::LaunchFirmwareApplet(u64 raw_program_id, std::optional<Service::NFP::CabinetMode> cabinet_mode) { auto const program_id = Service::AM::AppletProgramId(raw_program_id); @@ -4455,8 +4613,13 @@ void MainWindow::OnMouseActivity() { } } -void MainWindow::OnCheckFirmwareDecryption() { - if (!ContentManager::AreKeysPresent()) { +void MainWindow::OnCheckFirmwareDecryption(bool prompt) { + if (!ContentManager::AreKeysPresent() && !prompt) { + // At startup only log it: loose ExeFS/RomFS dumps (all Switch 2 games so far) need no + // keys, and a modal question blocked every unattended launch. Tools > Install Decryption + // Keys still installs them. + LOG_WARNING(Frontend, "Decryption keys are missing: NCA, NSP and XCI games need them"); + } else if (!ContentManager::AreKeysPresent()) { const auto res = QtCommon::Frontend::Warning( tr("Derivation Components Missing"), tr("Decryption keys are missing. Install them now?"), diff --git a/src/yuzu/main_window.h b/src/yuzu/main_window.h index 9e15177841..290ecac6fb 100644 --- a/src/yuzu/main_window.h +++ b/src/yuzu/main_window.h @@ -13,6 +13,7 @@ #include <QElapsedTimer> #include <QMainWindow> #include <QMessageBox> +#include <QFutureWatcher> #include <QPushButton> #include <QTimer> #include <QTranslator> @@ -20,6 +21,7 @@ #include "common/common_types.h" #include "common/settings_enums.h" +#include "core/file_sys/nup_writer.h" #include "frontend_common/content_manager.h" #include "input_common/drivers/tas_input.h" #include "qt_common/config/qt_config.h" @@ -353,6 +355,7 @@ private slots: void OnGameListSetPlayTime(u64 program_id); void OnGameListDumpRomFS(u64 program_id, const std::string& game_path, DumpRomFSTarget target); void OnGameListVerifyIntegrity(const std::string& game_path); + void OnGameListConvertToNup(const std::string& game_path); void OnGameListCopyTID(u64 program_id); void OnGameListCreateShortcut(u64 program_id, const std::string& game_path, const QtCommon::Game::ShortcutTarget target); @@ -422,7 +425,8 @@ private slots: void OnCreateHomeMenuDesktopShortcut(); void OnCreateHomeMenuApplicationMenuShortcut(); void OnCaptureScreenshot(); - void OnCheckFirmwareDecryption(); + /// Checks for decryption keys; `prompt` asks to install them when they are missing. + void OnCheckFirmwareDecryption(bool prompt = true); #ifdef __unix__ void OnCheckGraphicsBackend(); #endif @@ -573,6 +577,11 @@ private: // Install progress dialog QProgressDialog* install_progress = nullptr; + QProgressDialog* nup_progress = nullptr; + QFutureWatcher<FileSys::Nup::PackResult> nup_watcher; + std::shared_ptr<std::atomic_bool> nup_cancel; + bool nup_running{}; + std::string nup_source_path; // Last game booted, used for multi-process apps QString last_filename_booted; diff --git a/src/yuzu_cmd/yuzu.cpp b/src/yuzu_cmd/yuzu.cpp index 4d71a29993..bc4002d8af 100644 --- a/src/yuzu_cmd/yuzu.cpp +++ b/src/yuzu_cmd/yuzu.cpp @@ -12,7 +12,9 @@ #include <cstdlib> #include <chrono> #include <cstdio> +#include <csignal> #include <cstring> +#include <filesystem> #include <iostream> #include <memory> #include <mutex> @@ -36,6 +38,7 @@ #include "core/core_timing.h" #include "core/cpu_manager.h" #include "core/crypto/key_manager.h" +#include "core/file_sys/nup_writer.h" #include "core/file_sys/registered_cache.h" #include "core/file_sys/vfs/vfs_real.h" #include "core/hle/service/am/applet_manager.h" @@ -248,6 +251,49 @@ struct SdlState { std::unique_ptr<EmuWindow_SDL3> emu_window; }; +static std::atomic_bool pack_nup_cancel{false}; + +// eden-cli --pack-nup <dump> [<out.nup>]; the output defaults to <dump>.nup next to the dump. +static bool PackNup(const std::string& source, const std::string& output) { + const auto to_path = [](const std::string& utf8) { + return std::filesystem::path{std::u8string{utf8.begin(), utf8.end()}}; + }; + const auto dump_root = to_path(source); + const auto archive = + output.empty() ? FileSys::Nup::DefaultArchivePath(dump_root) : to_path(output); + std::signal(SIGINT, [](int) { pack_nup_cancel = true; }); + + const auto start = std::chrono::steady_clock::now(); + auto last_report = start; + std::string error; + const auto result = FileSys::Nup::PackDump( + dump_root, archive, {}, + [&](const FileSys::Nup::PackProgress& p) { + const auto now = std::chrono::steady_clock::now(); + if (now - last_report < std::chrono::seconds(1) && p.files_done != p.files_total) { + return; + } + last_report = now; + fmt::print(stderr, "\r{} / {} MiB, {} / {} files", p.bytes_done >> 20, + p.bytes_total >> 20, p.files_done, p.files_total); + }, + &pack_nup_cancel, &error); + const auto seconds = + std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count(); + fmt::print(stderr, "\n"); + switch (result) { + case FileSys::Nup::PackResult::Success: + fmt::print(stderr, "Wrote {} in {:.1f} s\n", archive.string(), seconds); + return true; + case FileSys::Nup::PackResult::Cancelled: + fmt::print(stderr, "Cancelled\n"); + return false; + default: + fmt::print(stderr, "Packing failed: {}\n", error); + return false; + } +} + extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) { #ifdef _WIN32 SetUnhandledExceptionFilter(WriteCrashDump); @@ -298,6 +344,10 @@ extern "C" SDL_AppResult SDL_AppInit(void **appstate, int argc, char **argv) { Core::ApplyLaunchParams(lp); + if (lp.pack_nup_source) { + return PackNup(*lp.pack_nup_source, lp.filepath) ? SDL_APP_SUCCESS : SDL_APP_FAILURE; + } + if (lp.filepath.empty()) { LOG_CRITICAL(Frontend, "Failed to load ROM: No ROM specified"); return SDL_APP_FAILURE; |
