nx2: let the host page out texture pool backing first

Switch 2 sparse blocks and GPU-only regions get guest memory when a game
writes them (Mario Kart World copies 2-4 GB of compressed textures into
its sparse texture pools) and the texture cache reads it again only when
it builds an image there. Every 30 s, mark that backing cold: its pages
leave the working set (VirtualUnlock on Windows, MADV_PAGEOUT on Linux)
so the OS compresses or pages them out before anything else when RAM is
short. The contents stay. Mario Kart World: 8.5 GB of RAM in use at five
minutes instead of 9.0 GB on a machine with RAM to spare, frame times
unchanged.

2 files changed, 26 insertions, 0 deletions

diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp
index 75a9da80e9..c0e7160dd4 100644
--- a/src/common/host_memory.cpp
+++ b/src/common/host_memory.cpp
@@ -790,6 +790,27 @@ void HostMemory::Protect(size_t virtual_offset, size_t length, MemoryPermission
 #endif
 }
 
+void HostMemory::HintCold(size_t physical_offset, size_t length,
+                          std::optional<size_t> virtual_offset) {
+    if (length == 0 || backing_base == nullptr) {
+        return;
+    }
+    const auto hint = [length](u8* address) {
+#ifdef _WIN32
+        // Unlocking pages that are not locked takes them out of the working set.
+        VirtualUnlock(address, length);
+#elif defined(__linux__) && defined(MADV_PAGEOUT)
+        madvise(address, length, MADV_PAGEOUT);
+#else
+        static_cast<void>(address);
+#endif
+    };
+    hint(backing_base + physical_offset);
+    if (virtual_offset && virtual_base != nullptr && *virtual_offset + length <= virtual_size) {
+        hint(virtual_base + *virtual_offset);
+    }
+}
+
 void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value) {
     std::memset(backing_base + physical_offset, fill_value, length);
 }
diff --git a/src/common/host_memory.h b/src/common/host_memory.h
index 8747a1850f..fe0938103c 100644
--- a/src/common/host_memory.h
+++ b/src/common/host_memory.h
@@ -54,6 +54,11 @@ public:
 
     void ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value);
 
+    /// Memory written once and read again rarely: its pages leave the working set (backing and,
+    /// when given, its view at virtual_offset), so the OS compresses or pages them out first when
+    /// it needs RAM. The contents stay; using the pages brings them back.
+    void HintCold(size_t physical_offset, size_t length, std::optional<size_t> virtual_offset);
+
     [[nodiscard]] u8* BackingBasePointer() noexcept {
         return backing_base;
     }