Merge pull request #291 from Djihads80/nx2-experimental

[Update] Bring the mainline release up to speed with the performance updates from nx2-experimental

5 files changed, 124 insertions, 17 deletions

diff --git a/src/common/fiber.cpp b/src/common/fiber.cpp
index 08558a0026..613ac97e22 100644
--- a/src/common/fiber.cpp
+++ b/src/common/fiber.cpp
@@ -25,8 +25,10 @@ struct Fiber::FiberImpl {
     FiberImpl() {}
 
     u32 canary_1 = CANARY_VALUE;
-    std::array<u8, DEFAULT_STACK_SIZE> stack;
-    std::array<u8, DEFAULT_STACK_SIZE> rewind_stack;
+    // Left uninitialized: the host gives a stack's pages memory only as they are used. Zero-filled
+    // std::arrays (and a second stack nothing switched to) made every fiber, one per guest thread,
+    // hold 4 MiB: ~750 MiB for title 1's ~190 threads.
+    std::unique_ptr<u8[]> stack;
     u32 canary_2 = CANARY_VALUE;
 
     boost::context::detail::fcontext_t context{};
@@ -38,7 +40,6 @@ struct Fiber::FiberImpl {
     std::shared_ptr<Fiber> previous_fiber;
 
     u8* stack_limit = nullptr;
-    u8* rewind_stack_limit = nullptr;
     bool is_thread_fiber = false;
     bool released = false;
 };
@@ -49,10 +50,10 @@ void Fiber::SetRewindPoint(std::function<void()>&& rewind_func) {
 
 Fiber::Fiber(std::function<void()>&& entry_point_func) : impl{std::make_unique<FiberImpl>()} {
     impl->entry_point = std::move(entry_point_func);
-    impl->stack_limit = impl->stack.data();
-    impl->rewind_stack_limit = impl->rewind_stack.data();
+    impl->stack.reset(new u8[DEFAULT_STACK_SIZE]);
+    impl->stack_limit = impl->stack.get();
     u8* stack_base = impl->stack_limit + DEFAULT_STACK_SIZE;
-    impl->context = boost::context::detail::make_fcontext(stack_base, impl->stack.size(), [](boost::context::detail::transfer_t transfer) -> void {
+    impl->context = boost::context::detail::make_fcontext(stack_base, DEFAULT_STACK_SIZE, [](boost::context::detail::transfer_t transfer) -> void {
         auto* fiber = static_cast<Fiber*>(transfer.data);
         ASSERT(fiber && fiber->impl && fiber->impl->previous_fiber && fiber->impl->previous_fiber->impl);
         ASSERT(fiber->impl->canary_1 == CANARY_VALUE);
diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp
index 147eba8477..2ef1982cbc 100644
--- a/src/common/host_memory.cpp
+++ b/src/common/host_memory.cpp
@@ -57,6 +57,7 @@
 
 #include "common/alignment.h"
 #include "common/assert.h"
+#include "common/div_ceil.h"
 #include "common/free_region_manager.h"
 #include "common/host_memory.h"
 #include "common/logging.h"
@@ -716,7 +717,9 @@ private:
 HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_)
     : backing_size(backing_size_)
     , virtual_size(virtual_size_)
+    , num_granules(Common::DivCeil(backing_size_, size_t{1} << GranuleShift))
 {
+    granules = std::make_unique<std::atomic<u8>[]>(num_granules);
 #if defined(__OPENORBIS__) || defined(__managarm__)
     LOG_WARNING(HW_Memory, "Platform doesn't support fastmem");
     backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
@@ -799,21 +802,98 @@ void HostMemory::Protect(size_t virtual_offset, size_t length, MemoryPermission
 #endif
 }
 
+void HostMemory::HintCold(size_t physical_offset, size_t length,
+                          std::optional<size_t> virtual_offset) const {
+    if (length == 0 || backing_base == nullptr) {
+        return;
+    }
+    const auto hint = [length](u8* address) {
+#ifdef _WIN32
+        // Unlocking pages that are not locked takes them out of the working set.
+        VirtualUnlock(address, length);
+#elif defined(__linux__) && defined(MADV_PAGEOUT)
+        madvise(address, length, MADV_PAGEOUT);
+#else
+        static_cast<void>(address);
+#endif
+    };
+    hint(backing_base + physical_offset);
+    if (virtual_offset && virtual_base != nullptr && *virtual_offset + length <= virtual_size) {
+        hint(virtual_base + *virtual_offset);
+    }
+}
+
 void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value) {
+    if (length == 0) {
+        return;
+    }
+    MarkUsed(physical_offset, length);
+    if (fill_value != 0) {
+        std::memset(backing_base + physical_offset, fill_value, length);
+        MarkDirty(physical_offset, length);
+        return;
+    }
+    // Fresh backing memory is zero and the host gives it no RAM until it's touched: clear only
+    // the granules that may hold data (title 1's heap made the host commit every page of it).
+    // A granule cleared in full is zero again.
 #ifdef __linux__
-    const size_t first = AlignUp(physical_offset, PageAlignment);
-    const size_t last = AlignDown(physical_offset + length, PageAlignment);
+    // Handing a whole granule back to the kernel frees its RAM; a memset would commit it.
     static const bool punch = [] {
         const char* value = std::getenv("EDEN_PUNCH_HOLE");
         return value == nullptr || value[0] != '0';
     }();
-    if (punch && fill_value == 0 && impl && last > first && impl->PunchHole(first, last - first)) {
-        std::memset(backing_base + physical_offset, 0, first - physical_offset);
-        std::memset(backing_base + last, 0, physical_offset + length - last);
+#endif
+    const size_t end = physical_offset + length;
+    for (size_t offset = physical_offset; offset < end;) {
+        const size_t index = offset >> GranuleShift;
+        const size_t granule_begin = index << GranuleShift;
+        const size_t granule_end = granule_begin + (size_t{1} << GranuleShift);
+        const size_t clear_end = (std::min)(granule_end, end);
+        if ((granules[index].load(std::memory_order_acquire) & GranuleDirty) != 0) {
+            const bool whole = offset == granule_begin && clear_end == granule_end;
+            bool cleared = false;
+#ifdef __linux__
+            cleared = whole && punch && impl && impl->PunchHole(granule_begin, granule_end - granule_begin);
+#endif
+            if (!cleared) {
+                std::memset(backing_base + offset, 0, clear_end - offset);
+            }
+            if (whole) {
+                granules[index].fetch_and(static_cast<u8>(~GranuleDirty), std::memory_order_release);
+            }
+        }
+        offset = clear_end;
+    }
+}
+
+void HostMemory::MarkUsed(size_t physical_offset, size_t length) {
+    if (length == 0) {
         return;
     }
-#endif
-    std::memset(backing_base + physical_offset, fill_value, length);
+    const size_t last = (physical_offset + length - 1) >> GranuleShift;
+    for (size_t index = physical_offset >> GranuleShift; index <= last; ++index) {
+        if ((granules[index].load(std::memory_order_relaxed) & GranuleUsed) == 0) {
+            granules[index].fetch_or(GranuleUsed, std::memory_order_release);
+        }
+    }
+}
+
+void HostMemory::MarkDirty(size_t physical_offset, size_t length) {
+    if (length == 0) {
+        return;
+    }
+    const size_t last = (physical_offset + length - 1) >> GranuleShift;
+    for (size_t index = physical_offset >> GranuleShift; index <= last; ++index) {
+        granules[index].fetch_or(GranuleDirty, std::memory_order_release);
+    }
+}
+
+void HostMemory::ForgetContents() {
+    for (size_t index = 0; index < num_granules; ++index) {
+        if ((granules[index].load(std::memory_order_relaxed) & GranuleUsed) != 0) {
+            granules[index].fetch_or(GranuleDirty, std::memory_order_release);
+        }
+    }
 }
 
 void HostMemory::EnableDirectMappedAddress() {
diff --git a/src/common/host_memory.h b/src/common/host_memory.h
index 8747a1850f..84315091c5 100644
--- a/src/common/host_memory.h
+++ b/src/common/host_memory.h
@@ -6,6 +6,7 @@
 
 #pragma once
 
+#include <atomic>
 #include <memory>
 #include <optional>
 #include "common/common_funcs.h"
@@ -52,8 +53,24 @@ public:
 
     void EnableDirectMappedAddress();
 
+    /// Clears guest memory being handed out. Memory still zero since it was created (or since its
+    /// last full clear) is skipped: clearing it only made the host give every page real memory.
     void ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value);
 
+    /// Memory written once and read again rarely: its pages leave the working set (backing and,
+    /// when given, its view at virtual_offset), so the OS compresses or pages them out first when
+    /// it needs RAM. The contents stay; using the pages brings them back.
+    void HintCold(size_t physical_offset, size_t length,
+                  std::optional<size_t> virtual_offset) const;
+    /// Pages the kernel hands out (cleared or not).
+    void MarkUsed(size_t physical_offset, size_t length);
+
+    /// Pages given back to the kernel: their contents are no longer known to be zero.
+    void MarkDirty(size_t physical_offset, size_t length);
+
+    /// A new kernel on this memory: everything a previous one used may hold data.
+    void ForgetContents();
+
     [[nodiscard]] u8* BackingBasePointer() noexcept {
         return backing_base;
     }
@@ -88,6 +105,12 @@ private:
     u8* backing_base{};
     u8* virtual_base{};
     size_t virtual_base_offset{};
+
+    static constexpr size_t GranuleShift = 16; ///< 64 KiB of backing per state byte
+    static constexpr u8 GranuleUsed = 1 << 0;   ///< Handed out at least once
+    static constexpr u8 GranuleDirty = 1 << 1;  ///< May hold non-zero data
+    std::unique_ptr<std::atomic<u8>[]> granules;
+    size_t num_granules{};
     // Windows requires it for kernels whom lack proper support for some functions!
     bool fallback_buffer{false};
 };
diff --git a/src/common/settings.h b/src/common/settings.h
index 97103f457d..74e2006d15 100644
--- a/src/common/settings.h
+++ b/src/common/settings.h
@@ -573,8 +573,10 @@ struct Values {
                                                                true,
                                                                true};
 
+    // BC3 by default: GPUs without ASTC otherwise keep ASTC textures as RGBA8, four times the
+    // VRAM (Mario Kart World: 7.3-7.8 GB of an 8 GB card, the rest paged into system RAM).
     SwitchableSetting<AstcRecompression, true> astc_recompression{linkage,
-                                                                  AstcRecompression::Uncompressed,
+                                                                  AstcRecompression::Bc3,
                                                                   "astc_recompression",
                                                                   Category::RendererAdvanced};
 
diff --git a/src/common/sparse_large_vector.cpp b/src/common/sparse_large_vector.cpp
index b96340999d..34aba73d78 100644
--- a/src/common/sparse_large_vector.cpp
+++ b/src/common/sparse_large_vector.cpp
@@ -39,12 +39,12 @@ static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) {
         std::lock_guard lock(vector_regions_mutex);
         for (auto region: vector_regions) {
             auto addr_shifted = exception_addr >> HostPageBits;
-            if (region.first <= addr_shifted && addr_shifted <= region.second) {
+            if (region.first <= addr_shifted && addr_shifted < region.second) {
                 addr = addr_shifted;
             }
 
             // Page-boundary accesses
-            if (auto addr_ = (exception_addr + 0x40) >> HostPageBits; addr_ != addr_shifted && region.first <= addr_ && addr_ <= region.second) {
+            if (auto addr_ = (exception_addr + 0x40) >> HostPageBits; addr_ != addr_shifted && region.first <= addr_ && addr_ < region.second) {
                 addr2 = addr_;
             }
 
@@ -164,7 +164,8 @@ void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept {
         return;
 #ifdef _WIN32
     std::lock_guard lock(vector_regions_mutex);
-    std::erase_if(vector_regions, [base](const auto& r) {return r.first == reinterpret_cast<u64>(base); });
+    // Regions are kept in host pages.
+    std::erase_if(vector_regions, [base](const auto& r) {return r.first == reinterpret_cast<u64>(base) >> HostPageBits; });
     ASSERT(VirtualFree(base, 0, MEM_RELEASE));
 #else
     ASSERT(munmap(base, size) == 0);