diff options
| author | kraibse <33869366+kraibse@users.noreply.github.com> | 2026-11-02 13:50:54 +0100 |
|---|---|---|
| committer | kraibse <33869366+kraibse@users.noreply.github.com> | 2026-11-02 14:43:01 +0100 |
| commit | c0cb3325e22756edb65c87d33ef298349ade0ad3 (patch) | |
| tree | e3f6d670018b57004915e69d1fc741b4efecb325 /src/core | |
| parent | 9210b8d69d172b5456e90cbd56b8f2bacbd7386b (diff) | |
| parent | a407703f6d29a5ab73328b4ededb12cf29b2758d (diff) | |
Merge nx2 into nx2-experimental
Resolved by area:
- audio feature_support.h: nx2's native revision handling.
- fiber.cpp: ours, which also drops the unused rewind stack.
- host_memory.cpp: ours granule bitmap and HintCold, plus nx2's
fallocate hole punching per whole dirty granule on Linux.
- texture_cache.h: nx2's old-age eviction pass, then our Collect().
- kepler_compute.cpp: shared upscale slots, then Fortune's Weave's,
then the Kirby Air Riders scan.
- nx2_gsv.h: ours, plus nx2's GetBufferMaxResolution and graphics
device ids (1-based).
- nvhost_ctrl_gpu.cpp: engine support flag bits 40-44.
- lower_to_maxwell.h: ours as the base, with nx2's seven lowering
commits applied one at a time. Second texture descriptor buffers
are read bindless in TEX, TLD.SCR (level zero) and SUST; FSEL
keeps its operand modifiers; TXD takes any channel mask.
lower_to_maxwell.h lowers 837 Fortune's Weave and 1183 Mario Kart
World shader dumps with the same rejects as nx2 and ours (12 and 0).
Assisted-by: Claude Sonnet 5.5 (conflict analysis and drafting)
Diffstat (limited to 'src/core')
| -rw-r--r-- | src/core/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | src/core/file_sys/nup_writer.cpp | 1459 | ||||
| -rw-r--r-- | src/core/file_sys/nup_writer.h | 60 | ||||
| -rw-r--r-- | src/core/hle/service/am/frontend/applet_software_keyboard.cpp | 8 | ||||
| -rw-r--r-- | src/core/hle/service/am/frontend/applet_software_keyboard_types.h | 60 | ||||
| -rw-r--r-- | src/core/hle/service/nvdrv/core/container.cpp | 2 | ||||
| -rw-r--r-- | src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp | 46 | ||||
| -rw-r--r-- | src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h | 2 | ||||
| -rw-r--r-- | src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp | 29 | ||||
| -rw-r--r-- | src/core/hle/service/nvdrv/devices/nx2_stub_device.h | 51 | ||||
| -rw-r--r-- | src/core/hle/service/nvnflinger/pixel_format.h | 1 | ||||
| -rw-r--r-- | src/core/hle/service/nx2_gsv.h | 286 | ||||
| -rw-r--r-- | src/core/launch_params.cpp | 8 | ||||
| -rw-r--r-- | src/core/launch_params.h | 2 |
14 files changed, 1953 insertions, 63 deletions
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index a9f4c2db7f..eeb3202d12 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -134,6 +134,8 @@ add_library(core STATIC file_sys/registered_cache.h file_sys/nup_container.cpp file_sys/nup_container.h + file_sys/nup_writer.cpp + file_sys/nup_writer.h file_sys/romfs.cpp file_sys/romfs.h file_sys/romfs_factory.cpp diff --git a/src/core/file_sys/nup_writer.cpp b/src/core/file_sys/nup_writer.cpp new file mode 100644 index 0000000000..8c7728ed01 --- /dev/null +++ b/src/core/file_sys/nup_writer.cpp @@ -0,0 +1,1459 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "core/file_sys/nup_writer.h" + +#include <algorithm> +#include <array> +#include <cctype> +#include <charconv> +#include <cstring> +#include <limits> +#include <map> +#include <memory> +#include <optional> +#include <span> +#include <stdexcept> +#include <unordered_map> +#include <unordered_set> +#include <vector> + +#include <nlohmann/json.hpp> +#include <openssl/evp.h> +#include <zstd.h> + +#include "common/fs/file.h" +#include "common/fs/path_util.h" +#include "core/file_sys/control_metadata.h" +#include "core/file_sys/vfs/vfs_real.h" +#include "core/loader/deconstructed_rom_directory.h" + +namespace FileSys::Nup { +namespace { + +using Json = nlohmann::json; +namespace fs = std::filesystem; + +constexpr std::array<u8, 4> Magic{'N', 'U', 'P', 0x01}; +constexpr std::array<u8, 4> TocMagic{'T', 'O', 'C', 0x01}; +constexpr u64 SuperblockSize = 0x80; +constexpr u64 TocHeaderSize = 0x30; +constexpr u64 PackageRecordSize = 0x40; +constexpr u64 DirRecordSize = 0x18; +constexpr u64 FileRecordSize = 0x50; +constexpr u64 ExtentAlignment = 16; +constexpr u32 NoDir = 0xFFFFFFFF; + +constexpr u8 KindBase = 0; +constexpr u8 KindUpdate = 1; +constexpr u8 KindAddon = 2; +constexpr u8 FlagStandaloneUpdate = 0x01; +constexpr u8 CodecRaw = 0; +constexpr u8 CodecZstd = 1; + +constexpr u16 TlvNpdm = 0x0001; +constexpr u16 TlvNacp = 0x0002; +constexpr u16 TlvIcon = 0x0003; +constexpr u16 TlvDisplayVersion = 0x0004; +constexpr u16 TlvTitleName = 0x0005; +constexpr u16 TlvPublisher = 0x0006; +constexpr u16 TlvSourceDump = 0x0008; + +constexpr int JournalVersion = 1; +constexpr u64 ProgressStep = 64ULL * 1024 * 1024; +constexpr size_t MaxChunkFiles = 4096; +constexpr size_t ReadBufferSize = 4 * 1024 * 1024; + +class PackError : public std::runtime_error { +public: + PackError(PackResult result_, const std::string& message) + : std::runtime_error(message), result{result_} {} + PackResult result; +}; + +u64 AlignUp(u64 value, u64 alignment) { + return (value + alignment - 1) / alignment * alignment; +} + +std::string ToUtf8(const fs::path& path) { + const auto u8 = path.generic_u8string(); + return {u8.begin(), u8.end()}; +} + +fs::path FromUtf8(const std::string& value) { + return fs::path{std::u8string{value.begin(), value.end()}}; +} + +std::string Lower(std::string value) { + std::ranges::transform(value, value.begin(), + [](unsigned char c) { return static_cast<char>(std::tolower(c)); }); + return value; +} + +fs::path WithSuffix(fs::path path, std::string_view suffix) { + path += suffix; + return path; +} + +bool IsFile(const fs::path& path) { + std::error_code ec; + return fs::is_regular_file(path, ec); +} + +fs::path FindDirectory(const fs::path& parent, std::string_view name) { + std::error_code ec; + if (parent.empty() || !fs::is_directory(parent, ec)) { + return {}; + } + const auto wanted = Lower(std::string{name}); + for (const auto& entry : fs::directory_iterator(parent, ec)) { + if (entry.is_directory(ec) && Lower(ToUtf8(entry.path().filename())) == wanted) { + return entry.path(); + } + } + return {}; +} + +std::optional<u64> ParseDecimal(const std::string& name) { + u64 value{}; + const auto* end = name.data() + name.size(); + const auto [ptr, ec] = std::from_chars(name.data(), end, value); + if (name.empty() || ec != std::errc{} || ptr != end) { + return std::nullopt; + } + return value; +} + +// The exefs folder a dump boots from: the newest updates/<version>, else base, else the root. +// Each must hold main, main.npdm and a romfs folder next to it or under base. +bool HasBootableExeFS(const fs::path& root) { + const auto bootable = [](const fs::path& exefs) { + return !exefs.empty() && IsFile(exefs / "main") && IsFile(exefs / "main.npdm"); + }; + fs::path exefs; + std::optional<u64> best_version; + if (const auto updates = FindDirectory(root, "updates"); !updates.empty()) { + std::error_code ec; + for (const auto& entry : fs::directory_iterator(updates, ec)) { + const auto version = ParseDecimal(ToUtf8(entry.path().filename())); + const auto candidate = FindDirectory(entry.path(), "exefs"); + if (version && bootable(candidate) && (!best_version || *version > *best_version)) { + best_version = version; + exefs = candidate; + } + } + } + const auto base = FindDirectory(root, "base"); + if (exefs.empty()) { + exefs = FindDirectory(base, "exefs"); + } + if (exefs.empty()) { + exefs = FindDirectory(root, "exefs"); + } + if (!bootable(exefs)) { + return false; + } + return !FindDirectory(exefs.parent_path(), "romfs").empty() || + !FindDirectory(base, "romfs").empty(); +} + +std::optional<u64> ParseTitleId(const std::string& name) { + u64 value{}; + const auto* end = name.data() + name.size(); + const auto [ptr, ec] = std::from_chars(name.data(), end, value, 16); + if (name.size() != 16 || ec != std::errc{} || ptr != end) { + return std::nullopt; + } + return value; +} + +std::string ToHex(std::span<const u8> data) { + static constexpr char digits[] = "0123456789abcdef"; + std::string out; + out.reserve(data.size() * 2); + for (const u8 byte : data) { + out += digits[byte >> 4]; + out += digits[byte & 0xF]; + } + return out; +} + +std::vector<u8> FromHex(const std::string& hex) { + if (hex.size() % 2 != 0) { + throw PackError(PackResult::IoError, "malformed journal"); + } + std::vector<u8> out(hex.size() / 2); + for (size_t i = 0; i < out.size(); ++i) { + const auto [ptr, ec] = std::from_chars(hex.data() + i * 2, hex.data() + i * 2 + 2, out[i], 16); + if (ec != std::errc{} || ptr != hex.data() + i * 2 + 2) { + throw PackError(PackResult::IoError, "malformed journal"); + } + } + return out; +} + +std::vector<u8> ReadWholeFile(const fs::path& path) { + Common::FS::IOFile in{path, Common::FS::FileAccessMode::Read}; + if (!in.IsOpen()) { + return {}; + } + std::vector<u8> data(in.GetSize()); + if (in.ReadSpan(std::span{data}) != data.size()) { + return {}; + } + return data; +} + +// Same as pack.py: the program ID at ACI0 + 0x10, whatever the NPDM header revision. +std::optional<u64> NpdmProgramId(std::span<const u8> npdm) { + static constexpr std::array<u8, 4> aci0{'A', 'C', 'I', '0'}; + const auto it = std::search(npdm.begin(), npdm.end(), aci0.begin(), aci0.end()); + const auto offset = static_cast<size_t>(it - npdm.begin()); + if (it == npdm.end() || offset + 0x18 > npdm.size()) { + return std::nullopt; + } + u64 id = 0; + for (size_t i = 0; i < 8; ++i) { + id |= static_cast<u64>(npdm[offset + 0x10 + i]) << (8 * i); + } + return id; +} + +void PutLE(std::vector<u8>& buffer, size_t offset, u64 value, size_t bytes) { + for (size_t i = 0; i < bytes; ++i) { + buffer[offset + i] = static_cast<u8>(value >> (8 * i)); + } +} + +void PutBytes(std::vector<u8>& buffer, size_t offset, std::span<const u8> bytes) { + std::memcpy(buffer.data() + offset, bytes.data(), bytes.size()); +} + +using Digest = std::array<u8, 32>; + +class Sha256 { +public: + Sha256() : ctx{EVP_MD_CTX_new()} { + if (!ctx) { + throw PackError(PackResult::IoError, "cannot initialise SHA-256"); + } + Reset(); + } + ~Sha256() { + EVP_MD_CTX_free(ctx); + } + Sha256(const Sha256&) = delete; + Sha256& operator=(const Sha256&) = delete; + + void Reset() { + if (EVP_DigestInit_ex(ctx, EVP_sha256(), nullptr) != 1) { + throw PackError(PackResult::IoError, "cannot initialise SHA-256"); + } + } + + void Update(const void* data, size_t size) { + EVP_DigestUpdate(ctx, data, size); + } + + Digest Final() { + Digest digest{}; + unsigned int length{}; + EVP_DigestFinal_ex(ctx, digest.data(), &length); + return digest; + } + +private: + EVP_MD_CTX* ctx; +}; + +class ZstdContext { +public: + // Archives must be byte-identical across platforms, so compression is always + // single-threaded (zstd's multithreaded frames differ) and every frame parameter is set + // explicitly instead of relying on library defaults. + explicit ZstdContext(int level) : cctx{ZSTD_createCCtx()} { + if (!cctx || ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level)) || + ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0)) || + ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 1)) || + ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0)) || + ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_dictIDFlag, 0))) { + throw PackError(PackResult::IoError, "cannot initialise zstd"); + } + } + ~ZstdContext() { + ZSTD_freeCCtx(cctx); + } + ZstdContext(const ZstdContext&) = delete; + ZstdContext& operator=(const ZstdContext&) = delete; + + // The pledged size makes zstd record the content size in the frame header, which the + // format requires. + void Begin(u64 size) { + if (ZSTD_isError(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only)) || + ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(cctx, size))) { + throw PackError(PackResult::IoError, "cannot start zstd frame"); + } + } + + ZSTD_CCtx* get() const { + return cctx; + } + +private: + ZSTD_CCtx* cctx; +}; + +// One package and the metadata it is written with. The plan is everything that must survive a +// low-storage run deleting its sources, so it is what the journal stores. +struct PlanPackage { + u8 kind{}; + u8 flags{}; + u64 title_id{}; + u32 version{}; + std::vector<u8> meta; +}; + +struct PlanEntry { + u32 package{}; + std::string path; // inside the package tree, e.g. "romfs/Data/file.bin" + std::string source; // relative to the dump root + bool directory{}; +}; + +struct Plan { + std::vector<PlanPackage> packages; + std::vector<PlanEntry> entries; +}; + +struct Extent { + u64 size{}; + Digest sha256{}; + u64 offset{}; + u64 stored_size{}; + u8 codec{}; +}; + +struct Layout { + struct Dir { + std::string name; + u32 first_child{}; + u32 child_count{}; + }; + struct File { + std::string name; + size_t entry{}; + }; + struct Package { + u32 root_dir{}; + u32 exefs_dir = NoDir; + u32 romfs_dir = NoDir; + }; + std::vector<Dir> dirs; + std::vector<File> files; + std::vector<u64> children; // FileTag | file index, or dir index + std::vector<Package> packages; + std::string pool; + std::unordered_map<std::string, u32> pool_index; + u64 toc_size{}; + u64 blob_offset{}; + + static constexpr u64 FileTag = 1ULL << 32; + + u32 Intern(const std::string& name) { + const auto [it, inserted] = pool_index.try_emplace(name, static_cast<u32>(pool.size())); + if (inserted) { + pool += name; + } + return it->second; + } +}; + +// Children sorted by the bytes of their names: std::string compares as unsigned char. +struct TreeDir { + struct Child { + std::unique_ptr<TreeDir> dir; + size_t file_entry{}; + }; + std::string name; + std::map<std::string, Child> children; + + TreeDir* Descend(const std::string& component) { + auto& child = children[component]; + if (!child.dir) { + child.dir = std::make_unique<TreeDir>(); + child.dir->name = component; + } + return child.dir.get(); + } +}; + +// Lays out nodes as pack.py does: directories in pre-order, each directory's child slots +// contiguous, its files numbered in slot order before its subdirectories are visited. +u32 AssignDir(TreeDir& dir, Layout& layout) { + const auto id = static_cast<u32>(layout.dirs.size()); + layout.dirs.push_back({dir.name, static_cast<u32>(layout.children.size()), + static_cast<u32>(dir.children.size())}); + const size_t slot_base = layout.children.size(); + layout.children.resize(slot_base + dir.children.size()); + size_t slot = slot_base; + for (auto& [name, child] : dir.children) { + if (!child.dir) { + layout.children[slot] = Layout::FileTag | layout.files.size(); + layout.files.push_back({name, child.file_entry}); + } + ++slot; + } + slot = slot_base; + for (auto& [name, child] : dir.children) { + if (child.dir) { + layout.children[slot] = AssignDir(*child.dir, layout); + } + ++slot; + } + return id; +} + +Layout BuildLayout(const Plan& plan) { + std::vector<TreeDir> roots(plan.packages.size()); + for (size_t i = 0; i < plan.entries.size(); ++i) { + const auto& entry = plan.entries[i]; + TreeDir* dir = &roots.at(entry.package); + size_t start = 0; + while (true) { + const auto slash = entry.path.find('/', start); + const auto component = entry.path.substr(start, slash - start); + if (slash == std::string::npos) { + if (entry.directory) { + dir->Descend(component); + } else { + auto& child = dir->children[component]; + if (child.dir) { + throw PackError(PackResult::InvalidSource, + "a file and a folder share the name " + entry.path); + } + child.file_entry = i; + } + break; + } + dir = dir->Descend(component); + start = slash + 1; + } + } + + Layout layout; + for (auto& root : roots) { + Layout::Package package; + package.root_dir = AssignDir(root, layout); + const auto& root_dir = layout.dirs[package.root_dir]; + for (u32 slot = root_dir.first_child; slot < root_dir.first_child + root_dir.child_count; + ++slot) { + const u64 child = layout.children[slot]; + if (child & Layout::FileTag) { + continue; + } + const auto id = static_cast<u32>(child); + if (layout.dirs[id].name == "exefs") { + package.exefs_dir = id; + } else if (layout.dirs[id].name == "romfs") { + package.romfs_dir = id; + } + } + layout.packages.push_back(package); + } + + if (layout.dirs.size() + layout.files.size() >= NoDir) { + throw PackError(PackResult::InvalidSource, "the dump has too many files"); + } + for (const auto& dir : layout.dirs) { + layout.Intern(dir.name); + } + for (const auto& file : layout.files) { + layout.Intern(file.name); + } + if (layout.pool.size() > std::numeric_limits<u32>::max()) { + throw PackError(PackResult::InvalidSource, "the dump's file names are too long"); + } + + const u64 unpadded = TocHeaderSize + plan.packages.size() * PackageRecordSize + + layout.dirs.size() * DirRecordSize + layout.files.size() * FileRecordSize + + layout.children.size() * sizeof(u32) + layout.pool.size(); + layout.toc_size = AlignUp(unpadded, ExtentAlignment); + layout.blob_offset = AlignUp(SuperblockSize + layout.toc_size, ExtentAlignment); + return layout; +} + +std::vector<u8> SerializeTlvs(std::vector<std::pair<u16, std::vector<u8>>> records) { + std::ranges::stable_sort(records, {}, &std::pair<u16, std::vector<u8>>::first); + std::vector<u8> out; + for (const auto& [type, payload] : records) { + const size_t at = out.size(); + out.resize(at + 8 + payload.size()); + PutLE(out, at, type, 2); + PutLE(out, at + 4, payload.size(), 4); + PutBytes(out, at + 8, payload); + } + return out; +} + +std::vector<u8> StringBytes(const std::string& value) { + return {value.begin(), value.end()}; +} + +// The dump's own loader supplies control data and the display version, the same way a loose +// dump reports them when it boots (control.nacp beside it, else the RomFS's version tag). +std::unique_ptr<Loader::AppLoader_DeconstructedRomDirectory> OpenLoader( + RealVfsFilesystem& vfs, const fs::path& package_dir) { + const auto exefs = FindDirectory(package_dir, "exefs"); + if (exefs.empty() || !IsFile(exefs / "main")) { + return nullptr; + } + auto main = vfs.OpenFile(Common::FS::PathToUTF8String(exefs / "main"), OpenMode::Read); + if (!main) { + return nullptr; + } + return std::make_unique<Loader::AppLoader_DeconstructedRomDirectory>(std::move(main)); +} + +class Discovery { +public: + Discovery(const fs::path& dump_root_, std::vector<fs::path> excluded_) + : dump_root{dump_root_}, excluded{std::move(excluded_)} {} + + Plan Run() { + const auto base = FindDirectory(dump_root, "base"); + const auto updates = FindDirectory(dump_root, "updates"); + const auto dlc = FindDirectory(dump_root, "dlc"); + + std::optional<u64> base_title_id; + if (!base.empty() && HasContent(base)) { + base_title_id = AddPackage(KindBase, base, 0, {}); + } else if (updates.empty() && HasContent(dump_root)) { + base_title_id = AddPackage(KindBase, dump_root, 0, {"base", "updates", "dlc"}); + } + + std::vector<std::pair<u64, fs::path>> versions; + std::error_code ec; + if (!updates.empty()) { + for (const auto& entry : fs::directory_iterator(updates, ec)) { + const auto version = ParseDecimal(ToUtf8(entry.path().filename())); + if (version && entry.is_directory(ec) && HasContent(entry.path())) { + if (*version > std::numeric_limits<u32>::max()) { + throw PackError(PackResult::InvalidSource, + "update version out of range: " + ToUtf8(entry.path())); + } + versions.emplace_back(*version, entry.path()); + } + } + } + std::ranges::sort(versions); + const u8 update_flags = base_title_id ? 0 : FlagStandaloneUpdate; + for (const auto& [version, path] : versions) { + AddPackage(KindUpdate, path, static_cast<u32>(version), {}, update_flags, + base_title_id.value_or(0)); + } + + if (!dlc.empty()) { + std::vector<std::pair<std::string, fs::path>> addons; + for (const auto& title : fs::directory_iterator(dlc, ec)) { + const auto title_name = ToUtf8(title.path().filename()); + if (!ParseTitleId(title_name) || !title.is_directory(ec)) { + continue; + } + for (const auto& content : fs::directory_iterator(title.path(), ec)) { + if (content.is_directory(ec) && HasContent(content.path())) { + addons.emplace_back(title_name + "/" + ToUtf8(content.path().filename()), + content.path()); + } + } + } + std::ranges::sort(addons); + for (const auto& [name, path] : addons) { + AddPackage(KindAddon, path, 0, {}, 0, *ParseTitleId(name.substr(0, 16))); + } + } + + if (plan.packages.empty()) { + throw PackError(PackResult::InvalidSource, "no packages found in the dump"); + } + AddApplicationMetadata(); + for (size_t i = 0; i < plan.packages.size(); ++i) { + auto& tlvs = package_tlvs[i]; + tlvs.emplace_back(TlvSourceDump, StringBytes(ToUtf8(dump_root.filename()))); + plan.packages[i].meta = SerializeTlvs(std::move(tlvs)); + } + return std::move(plan); + } + +private: + static bool HasContent(const fs::path& dir) { + return !FindDirectory(dir, "exefs").empty() || !FindDirectory(dir, "romfs").empty(); + } + + // Returns the package's title ID: the NPDM program ID, else the folder's or base's. + u64 AddPackage(u8 kind, const fs::path& dir, u32 version, + const std::vector<std::string>& skip, u8 flags = 0, u64 fallback_id = 0) { + const auto index = static_cast<u32>(plan.packages.size()); + PlanPackage package{kind, flags, fallback_id, version, {}}; + auto& tlvs = package_tlvs.emplace_back(); + + const auto exefs = FindDirectory(dir, "exefs"); + const auto romfs = FindDirectory(dir, "romfs"); + fs::path npdm_path = exefs.empty() ? fs::path{} : exefs / "main.npdm"; + if (!IsFile(npdm_path)) { + npdm_path = dir / "main.npdm"; + } + if (IsFile(npdm_path)) { + auto npdm = ReadWholeFile(npdm_path); + if (const auto id = NpdmProgramId(npdm); kind != KindAddon && id && *id != 0) { + package.title_id = *id; + } + tlvs.emplace_back(TlvNpdm, std::move(npdm)); + } + + if (kind != KindAddon) { + if (auto loader = OpenLoader(vfs, dir)) { + std::string display_version; + if (loader->ReadDisplayVersion(display_version) == Loader::ResultStatus::Success && + !display_version.empty()) { + tlvs.emplace_back(TlvDisplayVersion, StringBytes(display_version)); + } + loaders.emplace_back(index, std::move(loader)); + } + } + plan.packages.push_back(std::move(package)); + + if (!exefs.empty()) { + CollectTree(index, exefs, "exefs"); + } + if (!romfs.empty()) { + CollectTree(index, romfs, "romfs"); + } + std::error_code ec; + for (const auto& entry : fs::directory_iterator(dir, ec)) { + const auto name = ToUtf8(entry.path().filename()); + const auto lower = Lower(name); + if (entry.path() == exefs || entry.path() == romfs || + std::ranges::find(skip, lower) != skip.end()) { + continue; + } + if (entry.is_directory(ec)) { + CollectTree(index, entry.path(), "data/" + name); + } else { + AddFile(index, entry, "data/" + name); + } + } + return plan.packages.back().title_id; + } + + // Names, icon and control data belong to the base package, or the first one without a base. + // Our dumps rarely carry a control folder; when one does, the base's own wins, then updates. + void AddApplicationMetadata() { + const auto app = static_cast<u32>(std::distance( + plan.packages.begin(), std::ranges::find(plan.packages, KindBase, &PlanPackage::kind))); + const u32 target = app < plan.packages.size() ? app : 0; + std::ranges::stable_sort(loaders, [target](const auto& a, const auto& b) { + return (a.first == target) > (b.first == target); + }); + + auto& tlvs = package_tlvs[target]; + bool have_control = false; + bool have_icon = false; + for (const auto& [index, loader] : loaders) { + NACP control; + if (!have_control && + loader->ReadControlData(control) == Loader::ResultStatus::Success) { + have_control = true; + tlvs.emplace_back(TlvNacp, control.GetRawBytes()); + if (const auto name = control.GetApplicationName(); !name.empty()) { + tlvs.emplace_back(TlvTitleName, StringBytes(name)); + } + if (const auto publisher = control.GetDeveloperName(); !publisher.empty()) { + tlvs.emplace_back(TlvPublisher, StringBytes(publisher)); + } + } + std::vector<u8> icon; + if (!have_icon && loader->ReadIcon(icon) == Loader::ResultStatus::Success && + !icon.empty()) { + have_icon = true; + tlvs.emplace_back(TlvIcon, std::move(icon)); + } + } + } + + bool IsExcluded(const fs::path& path) const { + const auto normal = fs::absolute(path).lexically_normal(); + return std::ranges::find(excluded, normal) != excluded.end(); + } + + void AddFile(u32 package, const fs::directory_entry& entry, const std::string& path) { + std::error_code ec; + if (entry.is_symlink(ec)) { + throw PackError(PackResult::InvalidSource, + "symlinks are not supported: " + ToUtf8(entry.path())); + } + if (!entry.is_regular_file(ec)) { + throw PackError(PackResult::InvalidSource, + "unsupported file type: " + ToUtf8(entry.path())); + } + // Dump-tool listings are excluded by the spec. + if (entry.path().filename() == ".listing" || IsExcluded(entry.path())) { + return; + } + plan.entries.push_back({package, path, Relative(entry.path()), false}); + } + + void CollectTree(u32 package, const fs::path& root, const std::string& prefix) { + plan.entries.push_back({package, prefix, Relative(root), true}); + for (auto it = fs::recursive_directory_iterator(root); + it != fs::recursive_directory_iterator(); ++it) { + const auto& entry = *it; + const auto path = prefix + "/" + ToUtf8(fs::relative(entry.path(), root)); + if (!entry.is_symlink() && entry.is_directory()) { + plan.entries.push_back({package, path, Relative(entry.path()), true}); + } else { + AddFile(package, entry, path); + } + } + } + + std::string Relative(const fs::path& path) const { + return ToUtf8(fs::relative(path, dump_root)); + } + + fs::path dump_root; + std::vector<fs::path> excluded; + RealVfsFilesystem vfs; + Plan plan; + std::vector<std::vector<std::pair<u16, std::vector<u8>>>> package_tlvs; + std::vector<std::pair<u32, std::unique_ptr<Loader::AppLoader_DeconstructedRomDirectory>>> + loaders; +}; + +Json PlanToJson(const Plan& plan) { + Json packages = Json::array(); + for (const auto& package : plan.packages) { + packages.push_back(Json{{"kind", package.kind}, + {"flags", package.flags}, + {"title_id", package.title_id}, + {"version", package.version}, + {"meta", ToHex(package.meta)}}); + } + Json entries = Json::array(); + for (const auto& entry : plan.entries) { + entries.push_back(Json{{"package", entry.package}, + {"path", entry.path}, + {"source", entry.source}, + {"directory", entry.directory}}); + } + return Json{{"format", "nup-journal"}, + {"version", JournalVersion}, + {"packages", packages}, + {"entries", entries}}; +} + +Plan PlanFromJson(const Json& json) { + if (json.at("format") != "nup-journal" || json.at("version") != JournalVersion) { + throw PackError(PackResult::IoError, "unsupported journal"); + } + Plan plan; + for (const auto& package : json.at("packages")) { + plan.packages.push_back({package.at("kind").get<u8>(), package.at("flags").get<u8>(), + package.at("title_id").get<u64>(), + package.at("version").get<u32>(), + FromHex(package.at("meta").get<std::string>())}); + } + for (const auto& entry : json.at("entries")) { + plan.entries.push_back({entry.at("package").get<u32>(), entry.at("path").get<std::string>(), + entry.at("source").get<std::string>(), + entry.at("directory").get<bool>()}); + } + return plan; +} + +void RemoveFile(const fs::path& path) { + std::error_code ec; + fs::remove(path, ec); +} + +bool IsEmptyDirectory(const fs::path& path) { + std::error_code ec; + return fs::is_directory(path, ec) && fs::is_empty(path, ec) && !ec; +} + +void RemoveEmptyTree(const fs::path& root) { + std::error_code ec; + std::vector<fs::path> directories; + for (auto it = fs::recursive_directory_iterator(root, ec); + !ec && it != fs::recursive_directory_iterator(); it.increment(ec)) { + if (it->is_directory(ec)) { + directories.push_back(it->path()); + } + } + for (auto it = directories.rbegin(); it != directories.rend(); ++it) { + if (IsEmptyDirectory(*it)) { + fs::remove(*it, ec); + } + } + if (IsEmptyDirectory(root)) { + fs::remove(root, ec); + } +} + +void RemoveEmptyParents(fs::path path, const fs::path& dump_root) { + while (!path.empty() && path.has_relative_path()) { + std::error_code ec; + if (fs::equivalent(path, dump_root, ec) || ec || !IsEmptyDirectory(path)) { + break; + } + fs::remove(path, ec); + path = path.parent_path(); + } +} + +class Packer { +public: + Packer(const fs::path& dump_root_, const fs::path& archive_, const PackOptions& options_, + const std::function<void(const PackProgress&)>& progress_, + const std::atomic_bool* cancel_) + : dump_root{dump_root_}, archive{archive_}, part{WithSuffix(archive_, ".part")}, + journal{WithSuffix(archive_, ".journal")}, options{options_}, progress_cb{progress_}, + cancel{cancel_} {} + + PackResult Run() { + std::error_code ec; + if (fs::exists(archive, ec)) { + throw PackError(PackResult::IoError, "archive already exists: " + ToUtf8(archive)); + } + // A journal without a complete plan line was cut short before anything was deleted. + const bool resume = + options.delete_sources && HasResumableJournal(archive) && LoadJournal(); + if (resume) { + OpenForResume(); + } else { + if (!IsLooseDump(dump_root)) { + throw PackError(PackResult::InvalidSource, + "expected exefs/main, exefs/main.npdm and a romfs folder"); + } + std::vector<fs::path> excluded; + for (const auto& path : {archive, part, journal}) { + excluded.push_back(fs::absolute(path).lexically_normal()); + } + plan = Discovery{dump_root, std::move(excluded)}.Run(); + layout = BuildLayout(plan); + OpenNew(); + } + extents.resize(layout.files.size()); + done.resize(layout.files.size()); + for (const auto& [index, extent] : journaled) { + extents[index] = extent; + done[index] = true; + Remember(extent); + } + CountTotals(); + Report(); + + for (size_t i = 0; i < layout.files.size(); ++i) { + if (done[i]) { + continue; + } + PackFile(i); + if (options.delete_sources && + (chunk_bytes >= options.chunk_bytes || chunk.size() >= MaxChunkFiles)) { + Checkpoint(); + } + } + if (options.delete_sources) { + Checkpoint(); + } + + Finish(); + out.Close(); + fs::rename(part, archive, ec); + if (ec) { + throw PackError(PackResult::IoError, "cannot rename archive: " + ec.message()); + } + + if (options.delete_sources) { + RemoveFile(journal); + RemoveSourceFolders(); + } + return PackResult::Success; + } + + void Abort() { + out.Close(); + if (!options.delete_sources) { + RemoveFile(part); + } + } + +private: + void OpenNew() { + RemoveFile(journal); + out.Open(part, Common::FS::FileAccessMode::Write); + if (!out.IsOpen()) { + throw PackError(PackResult::IoError, "cannot create " + ToUtf8(part)); + } + WriteZeros(layout.blob_offset); + cursor = layout.blob_offset; + if (options.delete_sources) { + AppendJournal(PlanToJson(plan).dump() + '\n'); + } + } + + bool LoadJournal() { + const auto contents = + Common::FS::ReadStringFromFile(journal, Common::FS::FileType::BinaryFile); + size_t start = 0; + bool have_plan = false; + while (start < contents.size()) { + const auto line_end = contents.find('\n', start); + if (line_end == std::string::npos) { + break; + } + const auto line = std::string_view{contents}.substr(start, line_end - start); + start = line_end + 1; + Json json; + try { + json = Json::parse(line); + } catch (const Json::exception&) { + // Only the tail can be torn: the journal is synced before sources are deleted. + break; + } + if (!have_plan) { + plan = PlanFromJson(json); + layout = BuildLayout(plan); + resume_end = layout.blob_offset; + have_plan = true; + continue; + } + const auto index = json.at("file").get<size_t>(); + const auto sha = FromHex(json.at("sha256").get<std::string>()); + if (index >= layout.files.size() || sha.size() != 32) { + throw PackError(PackResult::IoError, "malformed journal"); + } + Extent extent{json.at("size").get<u64>(), {}, json.at("offset").get<u64>(), + json.at("stored_size").get<u64>(), json.at("codec").get<u8>()}; + std::ranges::copy(sha, extent.sha256.begin()); + resume_end = std::max(resume_end, extent.offset + extent.stored_size); + journaled.emplace_back(index, extent); + } + return have_plan; + } + + void OpenForResume() { + const u64 end = resume_end; + out.Open(part, Common::FS::FileAccessMode::ReadWrite); + if (!out.IsOpen()) { + throw PackError(PackResult::IoError, "cannot open " + ToUtf8(part)); + } + if (out.GetSize() < end) { + throw PackError(PackResult::IoError, "partial archive is shorter than its journal; " + "delete " + + ToUtf8(part) + " and " + ToUtf8(journal) + + " to start over"); + } + if (!out.SetSize(end) || !out.Seek(static_cast<s64>(end))) { + throw PackError(PackResult::IoError, "cannot truncate " + ToUtf8(part)); + } + cursor = end; + + // Journaled before an interruption, but the delete never happened. + for (const auto& [index, extent] : journaled) { + RemoveFile(SourceOf(index)); + } + } + + fs::path SourceOf(size_t file_index) const { + return dump_root / FromUtf8(plan.entries[layout.files[file_index].entry].source); + } + + void CountTotals() { + for (size_t i = 0; i < layout.files.size(); ++i) { + ++progress.files_total; + if (done[i]) { + progress.bytes_total += extents[i].size; + progress.bytes_done += extents[i].size; + ++progress.files_done; + } else { + std::error_code ec; + const auto size = fs::file_size(SourceOf(i), ec); + if (ec) { + throw PackError(PackResult::InvalidSource, + "missing source file: " + ToUtf8(SourceOf(i))); + } + progress.bytes_total += size; + } + } + } + + void Remember(const Extent& extent) { + if (extent.size == 0) { + return; + } + sizes_seen.insert(extent.size); + by_hash.try_emplace(extent.sha256, extent); + } + + bool Cancelled() const { + return cancel && cancel->load(); + } + + void WriteZeros(u64 count) { + static constexpr std::array<u8, 4096> zeros{}; + while (count > 0) { + const auto n = static_cast<size_t>(std::min<u64>(count, zeros.size())); + Write(zeros.data(), n); + count -= n; + } + } + + void Write(const u8* data, size_t size) { + if (out.WriteSpan(std::span<const u8>{data, size}) != size) { + throw PackError(PackResult::IoError, "write failed, the disk may be full"); + } + } + + void AddProgress(u64 bytes) { + progress.bytes_done += bytes; + if (progress.bytes_done - last_report >= ProgressStep) { + Report(); + } + } + + // Reads the whole source through `sink`, checking that it still has the expected size. A file + // read more than once (hashed for dedup, or compressed and then stored raw) counts once. + template <typename Sink> + void ReadSource(Common::FS::IOFile& in, u64 size, Sink&& sink) { + if (!in.Seek(0)) { + throw PackError(PackResult::IoError, "cannot seek source"); + } + buffer.resize(ReadBufferSize); + u64 total = 0; + while (true) { + if (Cancelled()) { + throw PackError(PackResult::Cancelled, "cancelled"); + } + const auto read = in.ReadSpan(std::span{buffer}); + if (read == 0) { + break; + } + total += read; + if (total > size) { + break; + } + if (total > file_counted) { + AddProgress(total - file_counted); + file_counted = total; + } + if (!sink(buffer.data(), read)) { + return; + } + } + if (total != size) { + throw PackError(PackResult::IoError, "a source file changed while packing"); + } + } + + Digest HashSource(Common::FS::IOFile& in, u64 size) { + hash.Reset(); + ReadSource(in, size, [this](const u8* data, size_t n) { + hash.Update(data, n); + return true; + }); + return hash.Final(); + } + + ZstdContext& Compressor() { + if (!compressor) { + compressor = std::make_unique<ZstdContext>(options.zstd_level); + } + return *compressor; + } + + // Streams one zstd frame into the archive. Gives up (returns false) as soon as the output is + // no smaller than the input. + bool WriteCompressed(Common::FS::IOFile& in, u64 size, u64& stored) { + auto& zstd = Compressor(); + zstd.Begin(size); + out_buffer.resize(ZSTD_CStreamOutSize()); + stored = 0; + bool fits = true; + const auto drain = [&](ZSTD_inBuffer& input, ZSTD_EndDirective mode) { + while (true) { + ZSTD_outBuffer output{out_buffer.data(), out_buffer.size(), 0}; + const size_t remaining = ZSTD_compressStream2(zstd.get(), &output, &input, mode); + if (ZSTD_isError(remaining)) { + throw PackError(PackResult::IoError, + std::string{"zstd: "} + ZSTD_getErrorName(remaining)); + } + stored += output.pos; + if (stored >= size) { + return false; + } + Write(out_buffer.data(), output.pos); + const bool finished = mode == ZSTD_e_end ? remaining == 0 + : input.pos == input.size; + if (finished) { + return true; + } + } + }; + hash.Reset(); + ReadSource(in, size, [&](const u8* data, size_t n) { + hash.Update(data, n); + ZSTD_inBuffer input{data, n, 0}; + fits = drain(input, ZSTD_e_continue); + return fits; + }); + if (!fits) { + return false; + } + ZSTD_inBuffer empty{nullptr, 0, 0}; + return drain(empty, ZSTD_e_end); + } + + void PackFile(size_t index) { + const auto source = SourceOf(index); + Common::FS::IOFile in{source, Common::FS::FileAccessMode::Read}; + if (!in.IsOpen()) { + throw PackError(PackResult::IoError, "cannot read " + ToUtf8(source)); + } + Extent extent; + extent.size = in.GetSize(); + file_counted = 0; + + std::optional<Digest> known_hash; + if (extent.size == 0) { + known_hash = HashSource(in, 0); + } else if (sizes_seen.contains(extent.size)) { + // Only a file whose size matches an earlier extent can be a duplicate; hash it before + // writing anything so the shared extent costs no output. + known_hash = HashSource(in, extent.size); + if (const auto it = by_hash.find(*known_hash); + it != by_hash.end() && it->second.size == extent.size) { + extent = it->second; + Complete(index, extent); + return; + } + } + + if (extent.size == 0) { + extent.sha256 = *known_hash; + Complete(index, extent); + return; + } + + const u64 start = AlignUp(cursor, ExtentAlignment); + Seek(cursor); + WriteZeros(start - cursor); + extent.offset = start; + + bool compressed = false; + if (extent.size <= options.raw_threshold && options.zstd_level > 0) { + u64 stored = 0; + compressed = WriteCompressed(in, extent.size, stored); + if (compressed) { + extent.codec = CodecZstd; + extent.stored_size = stored; + extent.sha256 = hash.Final(); + } + } + if (!compressed) { + Seek(start); + hash.Reset(); + ReadSource(in, extent.size, [this](const u8* data, size_t n) { + hash.Update(data, n); + Write(data, n); + return true; + }); + extent.codec = CodecRaw; + extent.stored_size = extent.size; + extent.sha256 = hash.Final(); + } + cursor = start + extent.stored_size; + Remember(extent); + Complete(index, extent); + } + + void Seek(u64 offset) { + if (!out.Seek(static_cast<s64>(offset))) { + throw PackError(PackResult::IoError, "cannot seek " + ToUtf8(part)); + } + } + + void Complete(size_t index, const Extent& extent) { + extents[index] = extent; + done[index] = true; + if (options.delete_sources) { + chunk.push_back(index); + chunk_bytes += extent.size; + } + ++progress.files_done; + Report(); + } + + void AppendJournal(const std::string& lines) { + Common::FS::IOFile log{journal, Common::FS::FileAccessMode::Append}; + if (!log.IsOpen() || log.WriteString(lines) != lines.size() || !log.Commit()) { + throw PackError(PackResult::IoError, "cannot write " + ToUtf8(journal)); + } + } + + // Archive data first, then the journal, then the deletes, so an interruption loses nothing. + void Checkpoint() { + if (chunk.empty()) { + return; + } + if (!out.Commit()) { + throw PackError(PackResult::IoError, "cannot sync " + ToUtf8(part)); + } + std::string lines; + for (const auto index : chunk) { + const auto& extent = extents[index]; + lines += Json{{"file", index}, + {"size", extent.size}, + {"sha256", ToHex(extent.sha256)}, + {"offset", extent.offset}, + {"stored_size", extent.stored_size}, + {"codec", extent.codec}} + .dump(); + lines += '\n'; + } + AppendJournal(lines); + for (const auto index : chunk) { + RemoveFile(SourceOf(index)); + } + chunk.clear(); + chunk_bytes = 0; + } + + void Finish() { + // Meta blocks follow the last extent, as pack.py writes them. + Seek(cursor); + std::vector<std::pair<u64, u32>> meta; + for (const auto& package : plan.packages) { + if (package.meta.size() > std::numeric_limits<u32>::max()) { + throw PackError(PackResult::IoError, "package metadata is too large"); + } + meta.emplace_back(package.meta.empty() ? 0 : cursor, + static_cast<u32>(package.meta.size())); + Write(package.meta.data(), package.meta.size()); + cursor += package.meta.size(); + } + const u64 file_size = cursor; + + const auto toc = BuildToc(meta); + std::array<u8, 32> toc_hash{}; + unsigned int hash_length{}; + EVP_Digest(toc.data(), toc.size(), toc_hash.data(), &hash_length, EVP_sha256(), nullptr); + + // UUID: SHA-256 over title_id (u64), version (u32) and kind (u8) of each package. + Sha256 uuid_hash; + for (const auto& package : plan.packages) { + std::vector<u8> identity(13); + PutLE(identity, 0, package.title_id, 8); + PutLE(identity, 8, package.version, 4); + identity[12] = package.kind; + uuid_hash.Update(identity.data(), identity.size()); + } + const auto uuid = uuid_hash.Final(); + + std::vector<u8> super(SuperblockSize); + PutBytes(super, 0x00, Magic); + PutLE(super, 0x04, SuperblockSize, 2); + PutLE(super, 0x06, 1, 2); + PutLE(super, 0x08, SuperblockSize, 8); + PutLE(super, 0x10, toc.size(), 8); + PutBytes(super, 0x18, toc_hash); + PutLE(super, 0x38, layout.blob_offset, 8); + PutLE(super, 0x40, file_size, 8); + PutLE(super, 0x48, plan.packages.size(), 4); + PutBytes(super, 0x50, std::span{uuid}.first(16)); + + Seek(0); + Write(super.data(), super.size()); + Write(toc.data(), toc.size()); + if (!out.SetSize(file_size) || !out.Commit()) { + throw PackError(PackResult::IoError, "cannot finish " + ToUtf8(part)); + } + } + + // Every field is written little-endian at its documented offset. The package record is + // 0x40 bytes with 20 reserved bytes at 0x2C, as NupContainer reads it. + std::vector<u8> BuildToc(const std::vector<std::pair<u64, u32>>& meta) const { + std::vector<u8> toc(layout.toc_size); + const u64 dir_count = layout.dirs.size(); + PutBytes(toc, 0x00, TocMagic); + PutLE(toc, 0x04, plan.packages.size(), 4); + PutLE(toc, 0x08, dir_count, 8); + PutLE(toc, 0x10, layout.files.size(), 8); + + size_t pos = TocHeaderSize; + for (size_t i = 0; i < plan.packages.size(); ++i) { + const auto& package = plan.packages[i]; + const auto& record = layout.packages[i]; + PutLE(toc, pos + 0x00, package.title_id, 8); + PutLE(toc, pos + 0x08, record.root_dir, 4); + PutLE(toc, pos + 0x0C, record.exefs_dir, 4); + PutLE(toc, pos + 0x10, record.romfs_dir, 4); + PutLE(toc, pos + 0x14, package.version, 4); + toc[pos + 0x18] = package.kind; + toc[pos + 0x19] = package.flags; + PutLE(toc, pos + 0x20, meta[i].first, 8); + PutLE(toc, pos + 0x28, meta[i].second, 4); + pos += PackageRecordSize; + } + for (const auto& dir : layout.dirs) { + PutLE(toc, pos + 0x00, layout.pool_index.at(dir.name), 4); + PutLE(toc, pos + 0x04, dir.name.size(), 4); + PutLE(toc, pos + 0x08, dir.first_child, 4); + PutLE(toc, pos + 0x0C, dir.child_count, 4); + pos += DirRecordSize; + } + for (size_t i = 0; i < layout.files.size(); ++i) { + const auto& file = layout.files[i]; + const auto& extent = extents[i]; + PutLE(toc, pos + 0x00, layout.pool_index.at(file.name), 4); + PutLE(toc, pos + 0x04, file.name.size(), 4); + PutLE(toc, pos + 0x08, extent.size, 8); + // Empty files have no extent: offset, size and codec are all zero. + if (extent.size != 0) { + PutLE(toc, pos + 0x10, extent.offset, 8); + PutLE(toc, pos + 0x18, extent.stored_size, 8); + toc[pos + 0x40] = extent.codec; + } + PutBytes(toc, pos + 0x20, extent.sha256); + pos += FileRecordSize; + } + for (const u64 child : layout.children) { + const u64 id = (child & Layout::FileTag) ? dir_count + static_cast<u32>(child) : child; + PutLE(toc, pos, id, 4); + pos += sizeof(u32); + } + PutLE(toc, 0x18, pos, 8); + PutLE(toc, 0x20, layout.pool.size(), 8); + std::memcpy(toc.data() + pos, layout.pool.data(), layout.pool.size()); + return toc; + } + + void RemoveSourceFolders() { + for (const auto& entry : plan.entries) { + const auto depth = std::ranges::count(entry.path, '/'); + const bool top = entry.directory && + (depth == 0 || (depth == 1 && entry.path.starts_with("data/"))); + if (top) { + const auto source = dump_root / FromUtf8(entry.source); + RemoveEmptyTree(source); + RemoveEmptyParents(source.parent_path(), dump_root); + } + } + } + + void Report() { + last_report = progress.bytes_done; + if (progress_cb) { + progress_cb(progress); + } + } + + fs::path dump_root; + fs::path archive; + fs::path part; + fs::path journal; + PackOptions options; + std::function<void(const PackProgress&)> progress_cb; + const std::atomic_bool* cancel; + + Plan plan; + Layout layout; + std::vector<Extent> extents; + std::vector<bool> done; + std::vector<std::pair<size_t, Extent>> journaled; + + struct DigestHash { + size_t operator()(const Digest& digest) const { + size_t value; + std::memcpy(&value, digest.data(), sizeof(value)); + return value; + } + }; + std::unordered_map<Digest, Extent, DigestHash> by_hash; + std::unordered_set<u64> sizes_seen; + + Common::FS::IOFile out; + u64 cursor = 0; + u64 resume_end = 0; + std::vector<size_t> chunk; + u64 chunk_bytes = 0; + std::vector<u8> buffer; + std::vector<u8> out_buffer; + Sha256 hash; + std::unique_ptr<ZstdContext> compressor; + PackProgress progress{}; + u64 last_report = 0; + u64 file_counted = 0; +}; + +} // namespace + +bool IsLooseDump(const fs::path& dir) { + return HasBootableExeFS(dir); +} + +std::optional<fs::path> DumpRootFromMain(const fs::path& main) { + // The Qt game list stores the exefs folder rather than exefs/main. + std::error_code ec; + const auto exefs = fs::is_directory(main, ec) ? main : main.parent_path(); + if (Lower(ToUtf8(exefs.filename())) != "exefs") { + return std::nullopt; + } + auto root = exefs.parent_path(); + const auto name = Lower(ToUtf8(root.filename())); + if (name == "base") { + root = root.parent_path(); + } else if (ParseDecimal(name) && Lower(ToUtf8(root.parent_path().filename())) == "updates") { + root = root.parent_path().parent_path(); + } + if (!IsLooseDump(root)) { + return std::nullopt; + } + return root; +} + +fs::path DefaultArchivePath(const fs::path& dump_root) { + auto root = dump_root; + if (!root.has_filename()) { + root = root.parent_path(); + } + return root.parent_path() / (root.filename().u8string() + u8".nup"); +} + +bool HasResumableJournal(const fs::path& archive) { + std::error_code ec; + return fs::exists(WithSuffix(archive, ".part"), ec) && + fs::exists(WithSuffix(archive, ".journal"), ec); +} + +PackResult PackDump(const fs::path& dump_root, const fs::path& archive, const PackOptions& options, + const std::function<void(const PackProgress&)>& progress, + const std::atomic_bool* cancel, std::string* error_out) { + Packer packer{dump_root, archive, options, progress, cancel}; + try { + return packer.Run(); + } catch (const PackError& e) { + packer.Abort(); + if (error_out) { + *error_out = e.what(); + } + return e.result; + } catch (const std::exception& e) { + packer.Abort(); + if (error_out) { + *error_out = e.what(); + } + return PackResult::IoError; + } +} + +} // namespace FileSys::Nup diff --git a/src/core/file_sys/nup_writer.h b/src/core/file_sys/nup_writer.h new file mode 100644 index 0000000000..231387b156 --- /dev/null +++ b/src/core/file_sys/nup_writer.h @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <atomic> +#include <filesystem> +#include <functional> +#include <optional> +#include <string> + +#include "common/common_types.h" + +// Packs a loose dump into a NUP container. Every package is kept: base/, each +// updates/<version>/ and each dlc/<titleid>/<n>/. +namespace FileSys::Nup { + +struct PackProgress { + u64 bytes_done{}; + u64 bytes_total{}; + u64 files_done{}; + u64 files_total{}; +}; + +enum class PackResult { Success, Cancelled, InvalidSource, IoError }; + +// NupContainer keeps a zstd file in memory only while it fits its 512 MiB decompression cache; +// a larger one is decompressed again in full on every read, which makes big compressed RomFS +// files unusable. Files above this size are therefore stored raw, with headroom below the cap so +// that one file does not evict everything else from the cache. +constexpr u64 DefaultRawThreshold = 256ULL * 1024 * 1024; + +struct PackOptions { + // Low-storage mode: delete each source file once its extent is synced and journaled. + bool delete_sources = false; + u64 chunk_bytes = 256ULL * 1024 * 1024; + int zstd_level = 3; + // Files larger than this are always stored raw. + u64 raw_threshold = DefaultRawThreshold; +}; + +// A loose dump holds exefs/main and main.npdm, directly or under base/ or updates/<version>/, +// and a romfs folder. Folder names are matched case-insensitively. +bool IsLooseDump(const std::filesystem::path& dir); +// The dump folder a game list entry's exefs/main (or exefs folder) belongs to, if it is a loose +// dump. +std::optional<std::filesystem::path> DumpRootFromMain(const std::filesystem::path& main); +std::filesystem::path DefaultArchivePath(const std::filesystem::path& dump_root); + +// Writes <archive>.part and renames it when done; an existing archive is never overwritten. A +// cancelled or failed low-storage run keeps its .part and .journal, and calling this again with +// the same arguments resumes it. +PackResult PackDump(const std::filesystem::path& dump_root, const std::filesystem::path& archive, + const PackOptions& options = {}, + const std::function<void(const PackProgress&)>& progress = {}, + const std::atomic_bool* cancel = nullptr, std::string* error_out = nullptr); + +bool HasResumableJournal(const std::filesystem::path& archive); + +} // namespace FileSys::Nup diff --git a/src/core/hle/service/am/frontend/applet_software_keyboard.cpp b/src/core/hle/service/am/frontend/applet_software_keyboard.cpp index db42781e18..061394bff8 100644 --- a/src/core/hle/service/am/frontend/applet_software_keyboard.cpp +++ b/src/core/hle/service/am/frontend/applet_software_keyboard.cpp @@ -222,6 +222,14 @@ void SoftwareKeyboard::InitializeForeground() { sizeof(SwkbdConfigNew)); swkbd_config_common.use_utf8 = true; break; + case SwkbdAppletVersion::Version1441810: { + // Fire Emblem: Fortune's Weave asks for UTF-8 and keeps the reply up to the first NUL, so the + // Switch 1 layout's use_utf8 (a byte of the native initial_string_offset) left one letter. + const bool decoded = + DecodeNativeSwkbdConfig(swkbd_config_data, swkbd_config_common, swkbd_config_new); + ASSERT(decoded); + break; + } default: UNIMPLEMENTED_MSG("Unknown SwkbdConfig revision={} with size={}", swkbd_applet_version, swkbd_config_data.size()); diff --git a/src/core/hle/service/am/frontend/applet_software_keyboard_types.h b/src/core/hle/service/am/frontend/applet_software_keyboard_types.h index f263a704cd..8ff8391aa1 100644 --- a/src/core/hle/service/am/frontend/applet_software_keyboard_types.h +++ b/src/core/hle/service/am/frontend/applet_software_keyboard_types.h @@ -7,6 +7,9 @@ #pragma once #include <array> +#include <cstddef> +#include <cstring> +#include <span> #include "common/bit_field.h" #include "common/common_funcs.h" @@ -31,6 +34,7 @@ enum class SwkbdAppletVersion : u32_le { Version393227 = 0x6000B, // 6.0.0 - 7.0.1 Version524301 = 0x8000D, // 8.0.0+ Version1310736 = 0x140010, // Switch 2 (Pokemon Pokopia): 0x530-byte config, UTF-8 text + Version1441810 = 0x160012, // Switch 2 (Fire Emblem: Fortune's Weave): native 0x530-byte config }; enum class SwkbdType : u32 { @@ -184,8 +188,64 @@ struct SwkbdConfigNew { }; static_assert(sizeof(SwkbdConfigNew) == 0x4C8 - sizeof(SwkbdConfigCommon), "SwkbdConfigNew has incorrect size."); + +// SwkbdAppletVersion 0x160012: the text prefix of the common part keeps its layout, the flags, +// lengths, string offsets and dictionary fields move (checked against +// nn::swkbd::keyboardConfig2kbdConfig in Fire Emblem: Fortune's Weave's sdk). +struct SwkbdConfigNative { + std::array<u8, 0x3AA> common_prefix{}; + bool use_utf8{}; + bool use_blur_background{}; + u32 max_text_length{}; + u32 min_text_length{}; + SwkbdPasswordMode password_mode{}; + SwkbdTextDrawType text_draw_type{}; + u32 initial_string_offset{}; + u32 initial_string_length{}; + u32 user_dictionary_offset{}; + u32 user_dictionary_entries{}; + std::array<u64, 24> customized_dictionary_set_entries{}; + u8 total_customized_dictionary_set_entries{}; + bool use_text_check{}; + bool disable_cancel_button{}; + u8 trigger{}; + std::array<u32, 8> text_grouping{}; + INSERT_PADDING_BYTES(0x80); +}; +static_assert(sizeof(SwkbdConfigNative) == 0x530); +static_assert(offsetof(SwkbdConfigNative, use_text_check) == 0x48D); +static_assert(offsetof(SwkbdConfigNative, text_grouping) == 0x490); #pragma pack(pop) +inline bool DecodeNativeSwkbdConfig(std::span<const u8> data, SwkbdConfigCommon& common, + SwkbdConfigNew& config) { + if (data.size() != sizeof(SwkbdConfigNative)) { + return false; + } + SwkbdConfigNative native{}; + std::memcpy(&native, data.data(), sizeof(native)); + common = {}; + config = {}; + std::memcpy(static_cast<void*>(&common), native.common_prefix.data(), native.common_prefix.size()); + common.enable_return_button = native.common_prefix[0x1B] != 0; + common.use_utf8 = native.use_utf8; + common.use_blur_background = native.use_blur_background; + common.max_text_length = native.max_text_length; + common.min_text_length = native.min_text_length; + common.password_mode = native.password_mode; + common.text_draw_type = native.text_draw_type; + common.initial_string_offset = native.initial_string_offset; + common.initial_string_length = native.initial_string_length; + common.user_dictionary_offset = native.user_dictionary_offset; + common.user_dictionary_entries = native.user_dictionary_entries; + common.use_text_check = native.use_text_check; + config.text_grouping = native.text_grouping; + config.customized_dictionary_set_entries = native.customized_dictionary_set_entries; + config.total_customized_dictionary_set_entries = native.total_customized_dictionary_set_entries; + config.disable_cancel_button = native.disable_cancel_button; + return true; +} + struct SwkbdTextCheck { SwkbdTextCheckResult text_check_result{}; std::array<char16_t, STRING_BUFFER_SIZE / 2> text_check_message{}; diff --git a/src/core/hle/service/nvdrv/core/container.cpp b/src/core/hle/service/nvdrv/core/container.cpp index 9edce03f64..22db243e3f 100644 --- a/src/core/hle/service/nvdrv/core/container.cpp +++ b/src/core/hle/service/nvdrv/core/container.cpp @@ -11,6 +11,7 @@ #include "core/hle/service/nvdrv/core/heap_mapper.h" #include "core/hle/service/nvdrv/core/nvmap.h" #include "core/hle/service/nvdrv/core/syncpoint_manager.h" +#include "core/hle/service/nvdrv/devices/nx2_stub_device.h" #include "core/memory.h" #include "video_core/host1x/host1x.h" @@ -130,6 +131,7 @@ void Container::CloseSession(SessionId session_id) { session.has_preallocated_area = false; } session.is_active = false; + Devices::Nx2GpuMemoryRegistry::Get().ForgetDeviceRanges(session.asid.id); smmu.UnregisterProcess(impl->sessions[session_id.id].asid); impl->id_pool.emplace_front(session_id.id); } diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp index 86b5ed0677..40313a10fb 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp @@ -5,6 +5,7 @@ // SPDX-FileCopyrightText: 2021 Skyline Team and Contributors // SPDX-License-Identifier: GPL-3.0-or-later +#include <cstdlib> #include <cstring> #include <utility> @@ -35,6 +36,13 @@ nvhost_as_gpu::nvhost_as_gpu(Core::System& system_, Module& module_, NvCore::Con nvhost_as_gpu::~nvhost_as_gpu() { Nx2GpuMemoryRegistry::Get().RemoveListener(this); + // Leave shared ranges to the process's other address spaces; like this address space's own + // ranges, one nobody else maps stays allocated. + for (const auto& [key, range] : nx2_device_ranges) { + if (range.shared) { + Nx2GpuMemoryRegistry::Get().ReleaseDeviceRange(range.asid, key.first, key.second); + } + } } NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, std::span<const u8> input, @@ -134,25 +142,37 @@ void nvhost_as_gpu::Nx2Map(u64 gpu_address, u64 cpu_address, u64 size) { } // The same CPU range is often mapped twice (at its own address and in a sparse region), so // reuse its SMMU mapping. - DAddr device_address{}; - bool heap = false; + Nx2DeviceRange range{}; if (const auto it = nx2_device_ranges.find({cpu_address, size}); it != nx2_device_ranges.end()) { - device_address = it->second.address; - heap = it->second.heap; + range = it->second; } else if (session->has_preallocated_area && session->mapper->IsInBounds(cpu_address, size)) { - device_address = session->mapper->Map(cpu_address, size); - heap = true; + range = {session->mapper->Map(cpu_address, size), true}; } else { - auto& smmu = system.Host1x().MemoryManager(); - device_address = smmu.Allocate(Common::AlignUp(size, 0x20000)); - if (device_address == 0) { + const auto allocate = [&]() -> u64 { + auto& smmu = system.Host1x().MemoryManager(); + const DAddr address = smmu.Allocate(Common::AlignUp(size, 0x20000)); + if (address != 0) { + smmu.Map(address, cpu_address, size, session->asid, true); + } + return address; + }; + // EDEN_NX2_PRIVATE_DEVICE_RANGES=1 (A/B aid): a range per address space. + static const bool private_ranges = std::getenv("EDEN_NX2_PRIVATE_DEVICE_RANGES") != nullptr; + if (private_ranges) { + range = {allocate(), false}; + } else { + range = {Nx2GpuMemoryRegistry::Get().AcquireDeviceRange(session->asid.id, cpu_address, + size, allocate), + false, true, session->asid.id}; + } + if (range.address == 0) { LOG_CRITICAL(Service_NVDRV, "Switch 2 GPU memory {:#x}+{:#x}: out of SMMU space", cpu_address, size); return; } - smmu.Map(device_address, cpu_address, size, session->asid, true); } - nx2_device_ranges[{cpu_address, size}] = {device_address, heap}; + const DAddr device_address = range.address; + nx2_device_ranges[{cpu_address, size}] = range; Nx2GpuMemoryRegistry::Get().RecordDeviceRange(cpu_address, size, device_address); // On the GPU thread, like unmaps: a GMMU map takes the texture cache lock, and the callers hold // nx2:gpu locks that the GPU thread takes under that lock (sparse blocks backed on write). @@ -181,6 +201,10 @@ void nvhost_as_gpu::Nx2Release(u64 cpu_address, u64 size) { } const Nx2DeviceRange range = it->second; nx2_device_ranges.erase(it); + if (range.shared && + !Nx2GpuMemoryRegistry::Get().ReleaseDeviceRange(range.asid, cpu_address, size)) { + return; // another address space of the process still maps it + } auto* session = container.GetSession(nx2_session); auto* const mapper = range.heap && session != nullptr ? session->mapper.get() : nullptr; auto& smmu = system.Host1x().MemoryManager(); diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h index 39b63ea369..37b6d0d4c4 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h @@ -238,6 +238,8 @@ private: struct Nx2DeviceRange { DAddr address; bool heap; ///< Mapped through the session's heap mapper (else an SMMU allocation) + bool shared{}; ///< A reference to the process's range in Nx2GpuMemoryRegistry + size_t asid{}; }; std::map<std::pair<u64, u64>, Nx2DeviceRange> nx2_device_ranges; void Nx2Release(u64 cpu_address, u64 size); diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp index 69c0313228..93d51cd538 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp @@ -113,12 +113,22 @@ NvResult nvhost_ctrl_gpu::Ioctl3(DeviceFD fd, Ioctl command, std::span<const u8> case 0x6: return WrapFixedInlOut(this, &nvhost_ctrl_gpu::GetTPCMasks3, input, output, inline_output); - case 0x13: - LOG_DEBUG(Service_NVDRV, "(STUBBED) called."); - - // TODO (jarrodnorwell) - - return NvResult::NotImplemented; + case 0x13: { + // NVGPU_GPU_IOCTL_VSMS_MAPPING: an 8-byte argument struct plus a buffer of + // {gpc_index, tpc_index} byte pairs, one per virtual SM. The SDK treats a pair left at + // its 0xc0c0 sentinel as failure, so fill every pair: one GPC, two SMs per TPC. + if (command.raw != 0xc0084713 || input.size() != 8 || output.size() != 8 || + inline_output.size() < 2) { + return NvResult::BadParameter; + } + std::memcpy(output.data(), input.data(), output.size()); + for (size_t i = 0; i + 1 < inline_output.size(); i += 2) { + inline_output[i] = 0; // gpc_index + inline_output[i + 1] = static_cast<u8>(i / 4); // tpc_index + } + LOG_DEBUG(Service_NVDRV, "VSMS mapping for {} SMs", inline_output.size() / 2); + return NvResult::Success; + } default: break; } @@ -218,9 +228,10 @@ void ApplyNx2Characteristics(GC& gc) { if (remap) { gc.flags |= 1ULL << 35; } - // Engine support bits 40-44 (2D, 3D, compute, inline-to-memory, ZBC). NvRm's newer device - // path lists a class family in the device info only when its bit is set; without them a - // device opened that way has no compute class and Fire Emblem's CUDA driver reports no device. + // Engine support bits 40-44 (2D, 3D, compute, inline-to-memory, ZBC). The SDK filters its class + // enumeration by them, and NvRm's newer device path lists a class family in the device info + // only when its bit is set; without them Fire Emblem: Fortune's Weave aborts at boot and a + // device opened that way has no compute class (Fire Emblem's CUDA driver reports no device). gc.flags |= 0x1FULL << 40; // The Switch 2 layout keeps nine s16 interface versions at 0x70..0x80 (like nvgpu's diff --git a/src/core/hle/service/nvdrv/devices/nx2_stub_device.h b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h index 869e17caf6..8dee9ccfb7 100644 --- a/src/core/hle/service/nvdrv/devices/nx2_stub_device.h +++ b/src/core/hle/service/nvdrv/devices/nx2_stub_device.h @@ -5,6 +5,7 @@ #include <algorithm> #include <atomic> +#include <compare> #include <functional> #include <map> #include <mutex> @@ -166,12 +167,50 @@ public: { std::scoped_lock lk{device_mutex}; device_ranges.clear(); + shared_ranges.clear(); } std::scoped_lock lk{mutex}; mappings.clear(); sparse_regions.clear(); } + // Device (SMMU) range of a mapping outside the heap, shared by every address space of one + // process: the caches key memory by device address, so a range per address space made separate + // copies of the same memory, and presentation could read one the game never drew into. The + // first address space to map the range allocates it with `allocate` (0: out of space); every + // call takes a reference. + u64 AcquireDeviceRange(size_t asid, u64 cpu_address, u64 size, + const std::function<u64()>& allocate) { + std::scoped_lock lk{device_mutex}; + const SharedRangeKey key{asid, cpu_address, size}; + if (const auto it = shared_ranges.find(key); it != shared_ranges.end()) { + ++it->second.references; + return it->second.device_address; + } + const u64 device_address = allocate(); + if (device_address != 0) { + shared_ranges.emplace(key, SharedRange{device_address, 1}); + } + return device_address; + } + + // Drops a reference. True for the last one: the caller frees the range. + bool ReleaseDeviceRange(size_t asid, u64 cpu_address, u64 size) { + std::scoped_lock lk{device_mutex}; + const auto it = shared_ranges.find(SharedRangeKey{asid, cpu_address, size}); + if (it == shared_ranges.end() || --it->second.references != 0) { + return false; + } + shared_ranges.erase(it); + return true; + } + + // The process's session closed and its ASID can be reused: stop sharing its ranges. + void ForgetDeviceRanges(size_t asid) { + std::scoped_lock lk{device_mutex}; + std::erase_if(shared_ranges, [asid](const auto& entry) { return entry.first.asid == asid; }); + } + // Device (SMMU) address of a CPU range an address space mapped; used to present images. void RecordDeviceRange(u64 cpu_address, u64 size, u64 device_address) { std::scoped_lock lk{device_mutex}; @@ -197,6 +236,18 @@ private: std::mutex device_mutex; std::map<u64, std::pair<u64, u64>> device_ranges; // cpu start -> (size, device address) + struct SharedRangeKey { + size_t asid; + u64 cpu_address; + u64 size; + auto operator<=>(const SharedRangeKey&) const = default; + }; + struct SharedRange { + u64 device_address; + u32 references; + }; + std::map<SharedRangeKey, SharedRange> shared_ranges; + struct Mapping { u64 gpu_address; u64 cpu_address; diff --git a/src/core/hle/service/nvnflinger/pixel_format.h b/src/core/hle/service/nvnflinger/pixel_format.h index b6e8b5f8a0..a38c8be777 100644 --- a/src/core/hle/service/nvnflinger/pixel_format.h +++ b/src/core/hle/service/nvnflinger/pixel_format.h @@ -19,6 +19,7 @@ enum class PixelFormat : u32 { Bgra8888 = 5, Rgba5551 = 6, Rgba4444 = 7, + Rgba1010102 = 43, }; } // namespace Service::android diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h index f4e026a6db..37c797fa3f 100644 --- a/src/core/hle/service/nx2_gsv.h +++ b/src/core/hle/service/nx2_gsv.h @@ -21,6 +21,7 @@ #include <optional> #include <memory> #include <mutex> +#include <utility> #include <vector> #include "common/alignment.h" @@ -45,13 +46,17 @@ #include "core/hle/kernel/k_resource_limit.h" #include "core/hle/kernel/k_server_session.h" #include "core/hle/kernel/k_event.h" +#include "core/hle/kernel/k_hardware_timer.h" #include "core/hle/kernel/k_readable_event.h" #include "core/hle/kernel/k_session.h" +#include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/k_transfer_memory.h" +#include "core/hle/kernel/svc_results.h" #include "core/memory.h" #include "video_core/host1x/host1x.h" #include "video_core/host1x/nx2_video_decoder.h" #include "video_core/nx2.h" +#include "video_core/textures/decoders.h" #include "core/hle/service/hle_ipc.h" #include "core/hle/service/ipc_helpers.h" #include "core/hle/service/kernel_helpers.h" @@ -392,34 +397,6 @@ private: rb.Push(ResultSuccess); } - // nn::gpu::sdk::CreateGraphicsDevice(name, e.g. "NvRmGpu" in six words): the reply's first - // byte is the device id, the first free of 8 slots. The auto stub's reply left the request's - // name bytes there, so Fire Emblem's CUDA driver got device 0x47. - void CreateGraphicsDevice(HLERequestContext& ctx) { - u32 id = 0; - while (id < m_graphics_devices.size() && m_graphics_devices[id]) { - ++id; - } - if (id == m_graphics_devices.size()) { - LOG_ERROR(Service, "nx2:gpu TIPC 41: no free graphics device slot"); - IPC::ResponseBuilder rb{ctx, 2}; - rb.Push(ResultUnknown); - return; - } - m_graphics_devices[id] = true; - LOG_INFO(Service, "nx2:gpu TIPC 41 graphics device {}", id); - TipcReplyWords(ctx, {id}); - } - - void DestroyGraphicsDevice(HLERequestContext& ctx) { - const u32 id = ctx.CommandBuffer()[2]; - LOG_INFO(Service, "nx2:gpu TIPC 42 graphics device {}", id); - if (id < m_graphics_devices.size()) { - m_graphics_devices[id] = false; - } - TipcReplyWords(ctx, {}); - } - void GetCount33(HLERequestContext& ctx) { LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 33 [2]={:#x}, returning 1", ctx.CommandBuffer()[2]); TipcReplyWords(ctx, {1}); @@ -450,9 +427,39 @@ private: static_cast<u32>(second), static_cast<u32>(second >> 32)}); } + // An empty partition is min = ~0, max = 0 (min > max). Fire Emblem: Fortune's Weave's SDK + // reads (0, 0) as a one-byte partition at address 0 and aborts in nn::gpu::sdk. void GetGsvPartition(HLERequestContext& ctx) { LOG_WARNING(Service, "(STUBBED) nx2:gpu TIPC 202 [2]={:#x}, empty", ctx.CommandBuffer()[2]); - TipcReplyWords(ctx, {0, 0, 0, 0}); + TipcReplyWords(ctx, {~0U, ~0U, 0, 0}); + } + + // Fire Emblem: Fortune's Weave: 41(name u64, flags u64, debug label u64) returns a device id + // that later NvRm calls name; 0 is not a valid id. Only flags 0 has been seen. + void CreateGraphicsDevice(HLERequestContext& ctx) { + const auto* b = ctx.CommandBuffer(); + const u64 name = u64{b[2]} | (u64{b[3]} << 32); + const u64 flags = u64{b[4]} | (u64{b[5]} << 32); + u32 id = 0; + for (u32 candidate = 1; candidate < m_devices.size(); ++candidate) { + if (!m_devices[candidate]) { + m_devices[candidate] = true; + id = candidate; + break; + } + } + LOG_INFO(Service, "nx2:gpu 41 create graphics device name={:#x} flags={:#x} -> id {}", name, + flags, id); + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(id != 0 ? ResultSuccess : Kernel::ResultOutOfResource); + rb.Push(id); + } + + void DestroyGraphicsDevice(HLERequestContext& ctx) { + const u32 id = ctx.CommandBuffer()[2] & 0xff; + const bool existed = id < m_devices.size() && std::exchange(m_devices[id], false); + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(existed ? ResultSuccess : Kernel::ResultNotFound); } void OpenSubSession(HLERequestContext& ctx) { @@ -1367,9 +1374,9 @@ private: KernelHelpers::ServiceContext m_context; std::array<Kernel::KEvent*, NumIds> m_events{}; - std::array<bool, 8> m_graphics_devices{}; ///< Ids handed out by CreateGraphicsDevice std::map<u32, GpuMemory> m_memory; u32 m_next_memory_handle{1}; + std::array<bool, 64> m_devices{}; // gpu 41 graphics device ids (0 unused) // Every 30 s: the backing of sparse blocks and GPU-only regions is marked cold. The game // writes it once (texture uploads by 2D blits: title 1 fills 2-4 GB of texture pools) and it @@ -1428,6 +1435,10 @@ struct Nx2NativeObjects { u64 offset{}; // into its memory (images of a decoder pool share one) bool window_pool{}; bool decoder{}; // a video decoder output image (gsv 911) + // A decoder image the client reads on the GPU (gsv 428/426): written block-linear. + bool gpu_read{}; + u32 gpu_maps{}; // outstanding 426 maps + bool mapped_here{}; // 426 added its GPU mapping (no other mapping covered it) }; std::mutex mutex; std::map<u32, Memory> memories; @@ -1619,6 +1630,7 @@ public: {104, &INx2Vi2Session::QueryDisplayHdrStatus, "QueryDisplayHdrStatus"}, {105, &INx2Vi2Session::QueryDisplayHdrPaperWhite, "QueryDisplayHdrPaperWhite"}, {212, &INx2Vi2Session::GetLayerBufferBits, "GetLayerBufferBits"}, + {106, &INx2Vi2Session::GetBufferMaxResolution, "GetBufferMaxResolution"}, {228, &INx2Vi2Session::CancelClientBuffer, "CancelClientBuffer"}, {229, &INx2Vi2Session::PresentClientBuffer, "PresentClientBuffer"}, // Firmware commands without a known meaning: zeroed replies. @@ -1626,7 +1638,6 @@ public: {51, &INx2Vi2Session::StubReplyZeroed, "Unknown51"}, {101, &INx2Vi2Session::StubReplyZeroed, "Unknown101"}, {102, &INx2Vi2Session::StubReplyZeroed, "Unknown102"}, - {106, &INx2Vi2Session::StubReplyZeroed, "Unknown106"}, {107, &INx2Vi2Session::StubReplyZeroed, "Unknown107"}, {108, &INx2Vi2Session::StubReplyZeroed, "Unknown108"}, {111, &INx2Vi2Session::StubReplyZeroed, "Unknown111"}, @@ -1728,7 +1739,7 @@ private: const auto waited = std::chrono::steady_clock::now(); Composite(w[5], ~w[22], w[24], w[7]); const auto composited = std::chrono::steady_clock::now(); - PaceToVblank(); + PaceToVblank(w[12]); const auto paced = std::chrono::steady_clock::now(); // Where each frame's time goes on the present thread, averaged per 600 presents: the // game is blocked here while Eden's GPU thread catches up (fence) and composites. @@ -1784,18 +1795,39 @@ private: // present faster than that. Every slot is reported free here: hold each present until the // next vblank instead (title 4 ran at up to 160 fps, its timing and audio with it). std::chrono::steady_clock::time_point m_next_vblank{}; + u32 m_swap_word{}; - void PaceToVblank() { + // timing: present word 12, (swap interval << 16) | refresh rate. Fire Emblem: Fortune's Weave + // sends 0x1003c during its boot logos and 0x2003c (two vblanks per frame, 30 fps) after; paced + // at every vblank it ran at double speed. EDEN_NX2_SWAP_INTERVAL=<n> forces the interval. + void PaceToVblank(u32 timing) { if (!Settings::values.use_speed_limit.GetValue() || Settings::SpeedLimit() == 0) { return; } - const auto interval = std::chrono::nanoseconds{100'000'000'000LL / 60 / Settings::SpeedLimit()}; + static const u32 forced_interval = [] { + const char* value = std::getenv("EDEN_NX2_SWAP_INTERVAL"); + return value ? static_cast<u32>(std::strtoul(value, nullptr, 10)) : 0U; + }(); + const u32 swap_interval = + forced_interval != 0 ? forced_interval : std::clamp<u32>(timing >> 16, 1, 4); + const u32 refresh_rate = (timing & 0xffff) >= 30 && (timing & 0xffff) <= 240 ? timing & 0xffff : 60; + if (timing != std::exchange(m_swap_word, timing)) { + LOG_INFO(Service, "nx2:vi2 present #{} swap interval {} at {} Hz (word 12 = {:#x})", + m_presents, swap_interval, refresh_rate, timing); + } + const auto interval = std::chrono::nanoseconds{ + 100'000'000'000LL * swap_interval / refresh_rate / Settings::SpeedLimit()}; const auto now = std::chrono::steady_clock::now(); if (now > m_next_vblank + interval) { - // Running slower than 60 fps: show at once and don't bank the lost time. + // Running slower than the swap interval: show at once and don't bank the lost time. m_next_vblank = now; } else if (now < m_next_vblank) { - std::this_thread::sleep_until(m_next_vblank); + // Sleep this service thread in the guest kernel. It runs on guest core 3, and a host + // sleep held that core for the whole wait: game threads scheduled there ran only + // between presents (Fortune's Weave's loader read one RomFS block per frame). + const auto wait = std::chrono::duration_cast<std::chrono::nanoseconds>(m_next_vblank - now); + static_cast<void>(Kernel::GetCurrentThread(kernel).Sleep( + kernel, kernel.HardwareTimer().GetTick() + wait.count() + 2)); } m_next_vblank += interval; } @@ -1853,6 +1885,7 @@ private: void Composite(u32 image_id, u32 fence_id, u32 fence_value, u32 shown_size) { u64 address{}; Nx2ImageLayout layout{}; + u32 image_format{}; { std::scoped_lock lk{m_objects->mutex}; const auto image = m_objects->images.find(image_id); @@ -1865,6 +1898,7 @@ private: } address = memory->second.address; layout = Nx2CalculateImageLayout(image->second.preference.data()); + image_format = image->second.preference[6]; } const auto device = Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address); if (!device) { @@ -1924,7 +1958,9 @@ private: .width = layout.width, .height = layout.height, .stride = static_cast<u32>(layout.pitch / 4), - .pixel_format = android::PixelFormat::Rgba8888, + // Preference word 6: 1 = RGBA8, 4 = RGB10A2. + .pixel_format = image_format == 4 ? android::PixelFormat::Rgba1010102 + : android::PixelFormat::Rgba8888, .transform_flags = android::BufferTransformFlags::Unset, .crop_rect = {0, 0, static_cast<int>(crop_width), static_cast<int>(crop_height)}, .blending = Tegra::BlendMode::Opaque, @@ -2062,6 +2098,29 @@ private: TipcReplyWords(ctx, {1}); } + // The highest buffer resolution mode of the "default" display. Fire Emblem: Fortune's Weave picks + // its output from it: 1 -> 1280x720, 2 -> 1920x1080, 4 and 8 -> 2560x1440; 0 (the auto stub's + // reply), 3 and 5 fall back to 720p. Docked reports 4, handheld 2 (the screen is 1080p); + // EDEN_VI2_MAX_RESOLUTION_MODE=<n> overrides. + void GetBufferMaxResolution(HLERequestContext& ctx) { + const auto* packet = ctx.CommandBuffer(); + const u64 name = u64{packet[2]} | (u64{packet[3]} << 32); + constexpr u64 DefaultDisplay = 0x00746c7561666564ULL; // "default" + Result result = Kernel::ResultInvalidCombination; + u64 mode{}; + if (packet[0] == 0x7a && packet[1] == 2) { + result = Result{ErrorModule::VI, 7}; + if (name == DefaultDisplay) { + static const char* forced = std::getenv("EDEN_VI2_MAX_RESOLUTION_MODE"); + mode = forced ? std::strtoull(forced, nullptr, 0) : Settings::IsDockedMode() ? 4 : 2; + result = ResultSuccess; + } + } + IPC::ResponseBuilder rb{ctx, 4}; + rb.Push(result.raw); + rb.Push(mode); + } + // Must be < 8; nn::vi::detail::LayerHistory creates 7 timestamp recorders per layer. void GetTimestampRecorderCount(HLERequestContext& ctx) { LOG_WARNING(Service, "(STUBBED) nx2:vi2 TIPC 5, returning 7"); @@ -2621,6 +2680,7 @@ private: const Decoded& frame, std::vector<u8>& bytes) { u64 address{}; Nx2DecoderImageLayout layout{}; + bool block_linear{}; { std::scoped_lock lk{m_objects->mutex}; const auto image = m_objects->images.find(image_id); @@ -2628,6 +2688,8 @@ private: return {m_width, m_height}; } layout = Nx2CalculateDecoderImageLayout(image->second.preference.data()); + block_linear = image->second.gpu_read && + layout.format != Tegra::Host1x::Nx2PictureFormat::Nv12; const auto memory = m_objects->memories.find(image->second.memory); if (memory != m_objects->memories.end() && memory->second.address != 0) { address = memory->second.address + image->second.offset; @@ -2641,10 +2703,25 @@ private: const u32 width = std::min(frame.picture->width, layout.width); const u32 height = std::min(frame.picture->height, layout.height); bytes.resize(layout.size); + auto& memory = system.ApplicationMemory(); + if (block_linear) { + // Read on the GPU: RGBA (the blit's format 0xd5) in the native block-linear layout, + // 16-GOB blocks, as our Eden tree writes these images. + decoder->Render(*frame.picture, Tegra::Host1x::Nx2PictureFormat::Rgba8, width, height, + layout.pitch, 0, bytes); + m_tiled.resize(layout.size); + Tegra::Texture::SwizzleTexture(m_tiled, bytes, 4, static_cast<u32>(layout.pitch / 4), + layout.height, 1, 4, 0, 1, true); + memory.WriteBlock(address, m_tiled.data(), m_tiled.size()); + if (const auto device = + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address)) { + system.GPU().InvalidateRegion(*device, m_tiled.size()); + } + return {width, height}; + } decoder->Render(*frame.picture, layout.format, width, height, layout.pitch, layout.chroma_offset, bytes); DumpFrame(bytes, layout, width, height); - auto& memory = system.ApplicationMemory(); memory.WriteBlock(address, bytes.data(), layout.pitch * height); if (layout.format == Tegra::Host1x::Nx2PictureFormat::Nv12) { memory.WriteBlock(address + layout.chroma_offset, bytes.data() + layout.chroma_offset, @@ -2690,6 +2767,7 @@ private: u64 m_submitted{}; u32 m_syncpoint{NoSyncpoint}; ///< Counts finished frames (allocated at the first register) u32 m_sync_base{}; ///< Its value before the first frame + std::vector<u8> m_tiled; ///< Decode thread: block-linear copy of a frame (WriteImage) // Shared with the decode thread. std::mutex m_mutex; @@ -2733,6 +2811,7 @@ public: {429, &INx2GraphicsSystemSession::CreateNativeImage, "CreateNativeImage"}, {430, &INx2GraphicsSystemSession::DestroyNativeImage, "DestroyNativeImage"}, {426, &INx2GraphicsSystemSession::QueryNativeImage, "MapNativeImage"}, + {427, &INx2GraphicsSystemSession::UnmapNativeImage, "UnmapNativeImage"}, {428, &INx2GraphicsSystemSession::QueryNativeImage, "QueryNativeImage"}, {435, &INx2GraphicsSystemSession::CalculateNativeImageSize, "CalculateNativeImageSize"}, {1402, &INx2GraphicsSystemSession::OpenNvsrvSession, "OpenNvsrvSession (guessed)"}, @@ -3133,8 +3212,19 @@ private: void DestroyNativeMemory(HLERequestContext& ctx) { const u32 id = ctx.CommandBuffer()[2]; - std::scoped_lock lk{m_objects->mutex}; - m_objects->memories.erase(id); + std::vector<std::optional<std::pair<u64, u64>>> unmaps; + { + std::scoped_lock lk{m_objects->mutex}; + for (auto& [image_id, image] : m_objects->images) { + if (image.memory == id && image.mapped_here) { + unmaps.push_back(TakeDecoderMapping(image)); + } + } + m_objects->memories.erase(id); + } + for (const auto& range : unmaps) { + DropDecoderMapping(range); + } LOG_DEBUG(Service, "nx2:gsv 112 DestroyNativeMemory id {}", id); TipcReplyWords(ctx, {}); } @@ -3211,6 +3301,9 @@ private: // the 429 preference words (0x10, 0x14), a guess. void QueryNativeImage(HLERequestContext& ctx) { const u32 handle = ctx.CommandBuffer()[2]; + if (QueryDecoderImage(ctx, handle)) { + return; + } std::array<u32, 16> preference{}; u64 image_offset = 0; u64 memory_address = 0; @@ -3255,6 +3348,108 @@ private: } } + // 428/426 for a decoder output image (Fire Emblem: Fortune's Weave's movie player blits each + // one, block-linear RGBA, format 0xd5, block height 16 GOBs, into a texture of its own). The + // reply follows our Eden tree, where the movies play: words 3-4 the GPU address, 5-6 the last + // byte's address, then the descriptor {1, 0, dimensions, .., 0x01000001, 1, offset, size} + // (offset 0 once mapped). 426 maps the image at its own address for the GPU: decoder pool + // memory is process memory (114) that no GPU mapping covers, and the blit read zeros. + bool QueryDecoderImage(HLERequestContext& ctx, u32 handle) { + const bool map = ctx.GetCommand() == 426; + u64 address{}; + u64 size{}; + u64 offset{}; + u32 dimensions{}; + bool register_mapping = false; + { + std::scoped_lock lk{m_objects->mutex}; + const auto image = m_objects->images.find(handle); + if (image == m_objects->images.end() || !image->second.decoder) { + return false; + } + const auto memory = m_objects->memories.find(image->second.memory); + if (memory == m_objects->memories.end() || memory->second.address == 0) { + return false; + } + image->second.gpu_read = true; + offset = image->second.offset; + size = image->second.size; + dimensions = image->second.preference[2]; + if (map) { + address = memory->second.address + offset; + register_mapping = image->second.gpu_maps++ == 0 && + !Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DeviceAddressOf(address); + image->second.mapped_here |= register_mapping; + } + } + if (register_mapping) { + Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Register(address, size); + } + LOG_INFO(Service, "nx2:gsv {} decoder image {} at {:#x}+{:#x}{}", ctx.GetCommand(), handle, + address, size, register_mapping ? ", mapped for the GPU" : ""); + std::array<u32, 37> words{}; + if (map) { + const u64 last = address + size - 1; + words[0] = static_cast<u32>(address); + words[1] = static_cast<u32>(address >> 32); + words[2] = static_cast<u32>(last); + words[3] = static_cast<u32>(last >> 32); + } + u32* const desc = words.data() + 4; + desc[0] = 1; + desc[2] = dimensions; + desc[8] = 0x01000001; + desc[9] = 1; + desc[10] = map ? 0 : static_cast<u32>(offset); + desc[11] = static_cast<u32>(size); + IPC::ResponseBuilder rb{ctx, static_cast<u32>(2 + words.size())}; + rb.PushRaw<u32>(ResultSuccess.raw); + for (const u32 w : words) { + rb.Push<u32>(w); + } + return true; + } + + // 427(u32 handle, u32 address space, u64 address, u64 last address): undoes a 426. The last + // unmap of a decoder image drops the GPU mapping 426 made. + void UnmapNativeImage(HLERequestContext& ctx) { + const auto* w = ctx.CommandBuffer(); + const u32 handle = w[2]; + std::optional<std::pair<u64, u64>> unmap; + { + std::scoped_lock lk{m_objects->mutex}; + if (const auto image = m_objects->images.find(handle); + image != m_objects->images.end() && image->second.gpu_maps != 0 && + --image->second.gpu_maps == 0) { + unmap = TakeDecoderMapping(image->second); + } + } + LOG_DEBUG(Service, "nx2:gsv 427 image {} [{:#x}, {:#x}]", handle, Pair(w, 4), Pair(w, 6)); + DropDecoderMapping(unmap); + TipcReplyWords(ctx, {}); + } + + // The range 426 mapped for an image, if it did; called with m_objects->mutex held. + std::optional<std::pair<u64, u64>> TakeDecoderMapping(Nx2NativeObjects::Image& image) { + image.gpu_maps = 0; + if (!std::exchange(image.mapped_here, false)) { + return std::nullopt; + } + const auto memory = m_objects->memories.find(image.memory); + if (memory == m_objects->memories.end() || memory->second.address == 0) { + return std::nullopt; + } + return std::pair{memory->second.address + image.offset, image.size}; + } + + static void DropDecoderMapping(const std::optional<std::pair<u64, u64>>& range) { + if (range) { + auto& registry = Nvidia::Devices::Nx2GpuMemoryRegistry::Get(); + registry.Unmap(range->first, range->second); + registry.ReleaseBacking(range->first, range->second); + } + } + // 911(preference[16], u32 count, u32 memory id) (nn::codec::InitializeDecoderImagePool): // result, the id of the first of `count` decoder output images; the client names image i of // the pool id + i. They share the pool's memory, one image size apart. @@ -3313,8 +3508,15 @@ private: void DestroyNativeImage(HLERequestContext& ctx) { const u32 id = ctx.CommandBuffer()[2]; - std::scoped_lock lk{m_objects->mutex}; - m_objects->images.erase(id); + std::optional<std::pair<u64, u64>> unmap; + { + std::scoped_lock lk{m_objects->mutex}; + if (const auto image = m_objects->images.find(id); image != m_objects->images.end()) { + unmap = TakeDecoderMapping(image->second); + m_objects->images.erase(image); + } + } + DropDecoderMapping(unmap); TipcReplyWords(ctx, {}); } diff --git a/src/core/launch_params.cpp b/src/core/launch_params.cpp index 25fdd45fdb..6c436d0346 100644 --- a/src/core/launch_params.cpp +++ b/src/core/launch_params.cpp @@ -39,6 +39,7 @@ LaunchParams ParseLaunchParams(Core::System& system, int argc, char *argv[], wch {"null-render", no_argument, 0, 'n'}, {"singlecore", no_argument, 0, 's'}, {"filter", no_argument, 0, 'x'}, + {"pack-nup", required_argument, 0, 'N'}, {0, 0, 0, 0}, // clang-format on }; @@ -62,6 +63,9 @@ LaunchParams ParseLaunchParams(Core::System& system, int argc, char *argv[], wch case 'g': p.filepath = std::string(optarg); break; + case 'N': + p.pack_nup_source = std::string(optarg); + break; case 'i': { p.input_profile = std::string(optarg); break; @@ -196,7 +200,9 @@ void ApplyLaunchParams(LaunchParams const& lp) noexcept { "-p, --program Pass following string as arguments to executable\n" "-u, --user Select a specific user profile from 0 to 7\n" "-d, --debug Run the GDB stub on a port from 1 to 65535\n" - "-v, --version Output version information and exit\n", + "-v, --version Output version information and exit\n" + "--pack-nup <dump> [<out.nup>]" + " Pack a loose dump with all its updates and DLC into a NUP file and exit\n", lp.argv0 ); } diff --git a/src/core/launch_params.h b/src/core/launch_params.h index 4da30fc1c2..8d503770d8 100644 --- a/src/core/launch_params.h +++ b/src/core/launch_params.h @@ -23,6 +23,8 @@ struct LaunchParams { std::optional<int> selected_user{}; std::optional<u16> override_gdb_port{}; std::optional<std::string> log_filter{}; + // --pack-nup <dump> [<out.nup>]: packs a loose dump instead of booting a game (eden-cli). + std::optional<std::string> pack_nup_source{}; u16 port = Network::DefaultRoomPort; bool use_multiplayer = false; bool fullscreen = false; |
