file_sys: decompress large NUP zstd members once

31b901b93b only cached a zstd member while it fit the 512 MiB cache.
Anything larger was never cached, so every Read() allocated the full
uncompressed size and decompressed the whole extent again, which makes a
multi-GB RomFS read in small chunks unusably slow. Members between 256
and 512 MiB evicted each other on alternate reads, and Insert copied the
vector so memory briefly doubled. Two threads decompressing the same
member also left a duplicate LRU entry and a wrong byte count.

The writer emits one zstd frame per member, so there are no frame
boundaries to seek by. Members larger than half the cache are now
streamed once into a temporary file under the cache directory and read
from there, using bounded memory. The file is removed when the
container is released. Smaller members stay in the LRU, which now moves
the vector in and ignores a duplicate insert.

Assisted-by: Claude Sonnet 5.5 (implementation and commit message)

4 files changed, 205 insertions, 21 deletions

diff --git a/src/core/file_sys/nup_container.cpp b/src/core/file_sys/nup_container.cpp
index 0e485b44ec..28e4a6ac89 100644
--- a/src/core/file_sys/nup_container.cpp
+++ b/src/core/file_sys/nup_container.cpp
@@ -8,11 +8,15 @@
 #include <cstring>
 #include <list>
 #include <map>
+#include <random>
+#include <span>
 #include <unordered_map>
 
+#include <fmt/format.h>
 #include <openssl/evp.h>
 #include <zstd.h>
 
+#include "common/fs/path_util.h"
 #include "common/logging.h"
 #include "common/swap.h"
 #include "core/file_sys/vfs/vfs_offset.h"
@@ -142,8 +146,8 @@ bool Sha256(const u8* data, std::size_t size, std::array<u8, 0x20>& out) {
     return length == out.size();
 }
 
-/// A member file whose extent is zstd-compressed. Decompresses on first read
-/// and caches the result (LRU-evicted) unless the file exceeds the cache cap.
+/// A member file whose extent is zstd-compressed. Decompresses on first read and caches the
+/// result (LRU-evicted); a file over half the cache cap is decompressed once to a temporary file.
 class NupVfsFile final : public VfsFile {
 public:
     NupVfsFile(VirtualFile backing_, u64 extent_offset_, u64 extent_size_, u64 uncompressed_size_,
@@ -151,7 +155,7 @@ public:
                std::size_t cache_index_)
         : backing(std::move(backing_)), extent_offset(extent_offset_), extent_size(extent_size_),
           uncompressed_size(uncompressed_size_), name(std::move(name_)), cache(std::move(cache_)),
-          cache_index(cache_index_), cache_enabled(uncompressed_size <= NupDecompressCache::DecompressCacheCap) {}
+          cache_index(cache_index_), cache_enabled(uncompressed_size <= cache->cap / 2) {}
 
     std::string GetName() const override {
         return name;
@@ -183,6 +187,10 @@ public:
         }
         length = std::min(length, static_cast<std::size_t>(uncompressed_size) - offset);
 
+        if (!cache_enabled) {
+            return ReadSpilled(data, length, offset);
+        }
+
         {
             std::scoped_lock lock{cache->mutex};
             if (const auto* cached = cache->Find(cache_index)) {
@@ -198,9 +206,7 @@ public:
         }
         {
             std::scoped_lock lock{cache->mutex};
-            if (cache_enabled) {
-                cache->Insert(cache_index, scratch);
-            }
+            cache->Insert(cache_index, scratch);
             if (const auto* cached = cache->Find(cache_index)) {
                 std::memcpy(data, cached->data() + offset, length);
                 return length;
@@ -220,6 +226,88 @@ public:
     }
 
 private:
+    std::size_t ReadSpilled(u8* data, std::size_t length, std::size_t offset) const {
+        std::shared_ptr<NupSpillFile> spill;
+        {
+            std::scoped_lock lock{cache->mutex};
+            spill = cache->GetSpill(cache_index);
+        }
+        std::scoped_lock lock{spill->mutex};
+        if (!spill->attempted) {
+            spill->attempted = true;
+            DecompressToFile(*spill);
+        }
+        if (!spill->file.IsOpen() || !spill->file.Seek(static_cast<s64>(offset))) {
+            return 0;
+        }
+        return spill->file.ReadSpan(std::span{data, length});
+    }
+
+    void DecompressToFile(NupSpillFile& spill) const {
+        const auto fail = [&](const char* reason) {
+            LOG_ERROR(Loader, "NUP zstd decompression of {} to a temporary file failed: {}", name,
+                      reason);
+            spill.file.Close();
+            if (!spill.path.empty()) {
+                std::error_code ec;
+                std::filesystem::remove(spill.path, ec);
+            }
+        };
+        if (extent_size > backing->GetSize() || extent_offset > backing->GetSize() - extent_size) {
+            return fail("extent out of range");
+        }
+
+        std::random_device random;
+        std::error_code ec;
+        std::filesystem::create_directories(cache->spill_dir, ec);
+        spill.path =
+            cache->spill_dir / fmt::format("{:08x}{:08x}-{}.tmp", random(), random(), cache_index);
+        Common::FS::IOFile out{spill.path, Common::FS::FileAccessMode::Write};
+        if (!out.IsOpen()) {
+            return fail("cannot create the temporary file");
+        }
+
+        std::unique_ptr<ZSTD_DCtx, decltype(&ZSTD_freeDCtx)> dctx{ZSTD_createDCtx(),
+                                                                  &ZSTD_freeDCtx};
+        std::vector<u8> in_buffer(ZSTD_DStreamInSize());
+        std::vector<u8> out_buffer(ZSTD_DStreamOutSize());
+        u64 read = 0;
+        u64 written = 0;
+        bool frame_done = false;
+        while (!frame_done && read < extent_size) {
+            const auto chunk =
+                static_cast<std::size_t>(std::min<u64>(in_buffer.size(), extent_size - read));
+            if (backing->Read(in_buffer.data(), chunk,
+                              static_cast<std::size_t>(extent_offset + read)) != chunk) {
+                return fail("short read of the compressed data");
+            }
+            read += chunk;
+            ZSTD_inBuffer input{in_buffer.data(), chunk, 0};
+            ZSTD_outBuffer output{};
+            do {
+                output = {out_buffer.data(), out_buffer.size(), 0};
+                const std::size_t result = ZSTD_decompressStream(dctx.get(), &output, &input);
+                if (ZSTD_isError(result) || output.pos > uncompressed_size - written) {
+                    return fail("corrupt or oversized data");
+                }
+                if (out.WriteSpan(std::span<const u8>{out_buffer.data(), output.pos}) !=
+                    output.pos) {
+                    return fail("cannot write the temporary file");
+                }
+                written += output.pos;
+                frame_done = result == 0;
+            } while (!frame_done && (input.pos < input.size || output.pos == output.size));
+        }
+        if (!frame_done || written != uncompressed_size) {
+            return fail("size mismatch");
+        }
+        out.Close();
+        spill.file.Open(spill.path, Common::FS::FileAccessMode::Read);
+        if (!spill.file.IsOpen()) {
+            fail("cannot reopen the temporary file");
+        }
+    }
+
     std::vector<u8> Decompress() const {
         if (extent_size > backing->GetSize() || extent_offset > backing->GetSize() - extent_size) {
             return {};
@@ -251,6 +339,13 @@ private:
 
 } // namespace
 
+NupSpillFile::~NupSpillFile() {
+    file.Close();
+    if (!path.empty()) {
+        std::error_code ec;
+        std::filesystem::remove(path, ec);
+    }
+}
 
 bool NupContainer::IsNupFile(const VirtualFile& file) {
     if (file == nullptr || file->GetSize() < SuperblockSize) {
@@ -261,8 +356,13 @@ bool NupContainer::IsNupFile(const VirtualFile& file) {
     return magic == HeaderMagic;
 }
 
-NupContainer::NupContainer(VirtualFile file_)
+NupContainer::NupContainer(VirtualFile file_, std::size_t memory_cache_cap,
+                           std::filesystem::path spill_dir_)
     : file(std::move(file_)), cache(std::make_shared<NupDecompressCache>()) {
+    cache->cap = memory_cache_cap;
+    cache->spill_dir = spill_dir_.empty()
+                           ? Common::FS::GetEdenPath(Common::FS::EdenPath::CacheDir) / "nup"
+                           : std::move(spill_dir_);
     if (file == nullptr) {
         return;
     }
diff --git a/src/core/file_sys/nup_container.h b/src/core/file_sys/nup_container.h
index 118b137a7b..0980c25517 100644
--- a/src/core/file_sys/nup_container.h
+++ b/src/core/file_sys/nup_container.h
@@ -4,6 +4,7 @@
 #pragma once
 
 #include <array>
+#include <filesystem>
 #include <functional>
 #include <list>
 #include <memory>
@@ -14,19 +15,36 @@
 #include <vector>
 
 #include "common/common_types.h"
+#include "common/fs/file.h"
 #include "core/file_sys/vfs/vfs_types.h"
 #include "core/file_sys/vfs/vfs_vector.h"
 #include "core/loader/loader.h"
 
 namespace FileSys {
 
+/// A zstd member too large for the in-memory cache, decompressed once into a temporary file.
+struct NupSpillFile {
+    ~NupSpillFile();
+
+    std::mutex mutex;
+    std::filesystem::path path;
+    Common::FS::IOFile file;
+    bool attempted{};
+};
+
 /// LRU over decompressed zstd extents, shared by the files of one container.
 struct NupDecompressCache {
+    static constexpr std::size_t DefaultCap = 512 * 1024 * 1024;
+
+    using Entry = std::pair<std::size_t, std::vector<u8>>;
+
     std::mutex mutex;
-    std::list<std::pair<std::size_t, std::vector<u8>>> lru;
-    std::unordered_map<std::size_t, std::list<std::pair<std::size_t, std::vector<u8>>>::iterator>
-        index;
+    std::list<Entry> lru;
+    std::unordered_map<std::size_t, std::list<Entry>::iterator> index;
     std::size_t bytes{};
+    std::size_t cap{DefaultCap};
+    std::filesystem::path spill_dir;
+    std::unordered_map<std::size_t, std::shared_ptr<NupSpillFile>> spills;
 
     std::vector<u8>* Find(std::size_t file_index) {
         const auto it = index.find(file_index);
@@ -37,21 +55,31 @@ struct NupDecompressCache {
         return &it->second->second;
     }
 
+    /// Takes ownership of `data` only when it is inserted.
     void Insert(std::size_t file_index, std::vector<u8>& data) {
-        while (bytes + data.size() > DecompressCacheCap && !lru.empty()) {
+        if (index.contains(file_index)) {
+            return;
+        }
+        while (bytes + data.size() > cap && !lru.empty()) {
             bytes -= lru.back().second.size();
             index.erase(lru.back().first);
             lru.pop_back();
         }
-        if (bytes + data.size() > DecompressCacheCap) {
+        if (bytes + data.size() > cap) {
             return;
         }
-        lru.emplace_front(file_index, data);
-        index.emplace(file_index, lru.begin());
         bytes += data.size();
+        lru.emplace_front(file_index, std::move(data));
+        index.emplace(file_index, lru.begin());
     }
 
-    static constexpr std::size_t DecompressCacheCap = 512 * 1024 * 1024;
+    std::shared_ptr<NupSpillFile> GetSpill(std::size_t file_index) {
+        auto& spill = spills[file_index];
+        if (!spill) {
+            spill = std::make_shared<NupSpillFile>();
+        }
+        return spill;
+    }
 };
 
 /**
@@ -101,7 +129,11 @@ public:
     /// Recursively merges overlay into a copy of base; overlay paths win.
     static VirtualDir MergeShadowDirs(const VirtualDir& base, const VirtualDir& overlay);
 
-    explicit NupContainer(VirtualFile file_);
+    /// Zstd members larger than half of `memory_cache_cap` are decompressed to a temporary file
+    /// in `spill_dir` (the cache directory by default) instead of being held in memory.
+    explicit NupContainer(VirtualFile file_,
+                          std::size_t memory_cache_cap = NupDecompressCache::DefaultCap,
+                          std::filesystem::path spill_dir_ = {});
     ~NupContainer();
 
     Loader::ResultStatus GetStatus() const {
@@ -119,7 +151,8 @@ public:
     std::optional<std::size_t> FindBestUpdatePackage() const;
 
     /// Opens a member file by node index; raw extents are zero-copy views into
-    /// the container, zstd extents are lazily decompressed and cached.
+    /// the container, zstd extents are lazily decompressed, cached in memory
+    /// or, when large, in a temporary file.
     VirtualFile OpenFile(std::size_t file_index);
 
     VirtualDir GetExeFSDir(std::size_t package_index);
diff --git a/src/core/file_sys/nup_writer.h b/src/core/file_sys/nup_writer.h
index d2423dd1f4..dd7b7e0883 100644
--- a/src/core/file_sys/nup_writer.h
+++ b/src/core/file_sys/nup_writer.h
@@ -25,10 +25,9 @@ struct PackProgress {
 
 enum class PackResult { Success, Cancelled, InvalidSource, IoError };
 
-// NupContainer keeps a zstd file in memory only while it fits its 512 MiB decompression cache;
-// a larger one is decompressed again in full on every read, which makes big compressed RomFS
-// files unusable. Files above this size are therefore stored raw, with headroom below the cap so
-// that one file does not evict everything else from the cache.
+// NupContainer keeps a zstd file in memory only while it fits half of its 512 MiB decompression
+// cache; a larger one is decompressed to a temporary file on first read, which costs time and
+// disk space. Files above this size are therefore stored raw.
 constexpr u64 DefaultRawThreshold = 256ULL * 1024 * 1024;
 
 // Application metadata written to the base package (or the first package when there is no base).
diff --git a/src/tests/core/nup_writer.cpp b/src/tests/core/nup_writer.cpp
index c0230d8694..5dec0134cb 100644
--- a/src/tests/core/nup_writer.cpp
+++ b/src/tests/core/nup_writer.cpp
@@ -253,6 +253,58 @@ TEST_CASE("NUP packs base, updates and add-ons and reads back through NupContain
     REQUIRE_FALSE(by_name.contains(".listing"));
 }
 
+TEST_CASE("NUP reads zstd members above the memory cache through a temporary file", "[nup]") {
+    TempDir t;
+    const auto root = t.path / "Game";
+    MakeDump(root);
+    std::string large(300 * 1024, '\0');
+    u32 x = 7;
+    for (auto& c : large) {
+        x = x * 1664525 + 1013904223;
+        c = static_cast<char>('a' + (x >> 28));
+    }
+    Put(root / "Base/RomFS/large.bin", large);
+    const auto archive = t.path / "Game.nup";
+    FileSys::Nup::PackOptions options{.raw_threshold = 1024 * 1024};
+    REQUIRE(FileSys::Nup::PackDump(root, archive, options) == FileSys::Nup::PackResult::Success);
+    for (const auto& file : RawFiles(ReadAll(archive))) {
+        if (file.name == "large.bin") {
+            REQUIRE(file.codec == 1);
+        }
+    }
+
+    const auto spill_dir = t.path / "spill";
+    FileSys::RealVfsFilesystem real;
+    {
+        // Members above 32 KiB do not fit the 64 KiB cache budget and are spilled to disk.
+        NupContainer nup{real.OpenFile(archive.string(), FileSys::OpenMode::Read), 64 * 1024,
+                         spill_dir};
+        REQUIRE(nup.GetStatus() == Loader::ResultStatus::Success);
+        const auto file = nup.GetRomFSDir(0)->GetFile("large.bin");
+        REQUIRE(file != nullptr);
+        REQUIRE(file->GetSize() == large.size());
+
+        std::string chunk(4096, '\0');
+        for (size_t offset = 0; offset < large.size(); offset += chunk.size()) {
+            const size_t count =
+                file->Read(reinterpret_cast<u8*>(chunk.data()), chunk.size(), offset);
+            REQUIRE(count == std::min(chunk.size(), large.size() - offset));
+            REQUIRE(chunk.compare(0, count, large, offset, count) == 0);
+        }
+        for (const size_t offset : {size_t{0}, size_t{123456}, size_t{77}, large.size() - 5}) {
+            const size_t count = file->Read(reinterpret_cast<u8*>(chunk.data()), 100, offset);
+            REQUIRE(count == std::min<size_t>(100, large.size() - offset));
+            REQUIRE(chunk.compare(0, count, large, offset, count) == 0);
+        }
+        REQUIRE(file->Read(reinterpret_cast<u8*>(chunk.data()), 10, large.size()) == 0);
+
+        REQUIRE(std::distance(fs::directory_iterator(spill_dir), fs::directory_iterator{}) > 0);
+        REQUIRE(PackedTree(nup.GetRomFSDir(0)) == SourceTree(root / "Base/RomFS"));
+        REQUIRE(nup.VerifyIntegrity({}) == Loader::ResultStatus::Success);
+    }
+    REQUIRE(std::distance(fs::directory_iterator(spill_dir), fs::directory_iterator{}) == 0);
+}
+
 TEST_CASE("NUP marks updates standalone when the dump has no base", "[nup]") {
     TempDir t;
     const auto root = t.path / "Game";