Merge remote-tracking branch 'origin/fix/memory-skip-zero-clears' into test/linux-all-prs-plus

# Conflicts:
#	src/common/host_memory.h

2 files changed, 76 insertions, 1 deletions

diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp
index 2c45dfd3a5..b724bed1bd 100644
--- a/src/common/host_memory.cpp
+++ b/src/common/host_memory.cpp
@@ -56,6 +56,7 @@
 
 #include "common/alignment.h"
 #include "common/assert.h"
+#include "common/div_ceil.h"
 #include "common/free_region_manager.h"
 #include "common/host_memory.h"
 #include "common/logging.h"
@@ -707,7 +708,9 @@ private:
 HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_)
     : backing_size(backing_size_)
     , virtual_size(virtual_size_)
+    , num_granules(Common::DivCeil(backing_size_, size_t{1} << GranuleShift))
 {
+    granules = std::make_unique<std::atomic<u8>[]>(num_granules);
 #if defined(__OPENORBIS__) || defined(__managarm__)
     LOG_WARNING(HW_Memory, "Platform doesn't support fastmem");
     backing_base = static_cast<u8*>(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
@@ -812,7 +815,62 @@ void HostMemory::HintCold(size_t physical_offset, size_t length,
 }
 
 void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value) {
-    std::memset(backing_base + physical_offset, fill_value, length);
+    if (length == 0) {
+        return;
+    }
+    MarkUsed(physical_offset, length);
+    if (fill_value != 0) {
+        std::memset(backing_base + physical_offset, fill_value, length);
+        MarkDirty(physical_offset, length);
+        return;
+    }
+    // Fresh backing memory is zero and the host gives it no RAM until it's touched: clear only
+    // the granules that may hold data (title 1's heap made the host commit every page of it).
+    // A granule cleared in full is zero again.
+    const size_t end = physical_offset + length;
+    for (size_t offset = physical_offset; offset < end;) {
+        const size_t index = offset >> GranuleShift;
+        const size_t granule_begin = index << GranuleShift;
+        const size_t granule_end = granule_begin + (size_t{1} << GranuleShift);
+        const size_t clear_end = (std::min)(granule_end, end);
+        if ((granules[index].load(std::memory_order_acquire) & GranuleDirty) != 0) {
+            std::memset(backing_base + offset, 0, clear_end - offset);
+            if (offset == granule_begin && clear_end == granule_end) {
+                granules[index].fetch_and(static_cast<u8>(~GranuleDirty), std::memory_order_release);
+            }
+        }
+        offset = clear_end;
+    }
+}
+
+void HostMemory::MarkUsed(size_t physical_offset, size_t length) {
+    if (length == 0) {
+        return;
+    }
+    const size_t last = (physical_offset + length - 1) >> GranuleShift;
+    for (size_t index = physical_offset >> GranuleShift; index <= last; ++index) {
+        if ((granules[index].load(std::memory_order_relaxed) & GranuleUsed) == 0) {
+            granules[index].fetch_or(GranuleUsed, std::memory_order_release);
+        }
+    }
+}
+
+void HostMemory::MarkDirty(size_t physical_offset, size_t length) {
+    if (length == 0) {
+        return;
+    }
+    const size_t last = (physical_offset + length - 1) >> GranuleShift;
+    for (size_t index = physical_offset >> GranuleShift; index <= last; ++index) {
+        granules[index].fetch_or(GranuleDirty, std::memory_order_release);
+    }
+}
+
+void HostMemory::ForgetContents() {
+    for (size_t index = 0; index < num_granules; ++index) {
+        if ((granules[index].load(std::memory_order_relaxed) & GranuleUsed) != 0) {
+            granules[index].fetch_or(GranuleDirty, std::memory_order_release);
+        }
+    }
 }
 
 void HostMemory::EnableDirectMappedAddress() {
diff --git a/src/common/host_memory.h b/src/common/host_memory.h
index 4d8d15cf58..84315091c5 100644
--- a/src/common/host_memory.h
+++ b/src/common/host_memory.h
@@ -6,6 +6,7 @@
 
 #pragma once
 
+#include <atomic>
 #include <memory>
 #include <optional>
 #include "common/common_funcs.h"
@@ -52,6 +53,8 @@ public:
 
     void EnableDirectMappedAddress();
 
+    /// Clears guest memory being handed out. Memory still zero since it was created (or since its
+    /// last full clear) is skipped: clearing it only made the host give every page real memory.
     void ClearBackingRegion(size_t physical_offset, size_t length, u32 fill_value);
 
     /// Memory written once and read again rarely: its pages leave the working set (backing and,
@@ -59,6 +62,14 @@ public:
     /// it needs RAM. The contents stay; using the pages brings them back.
     void HintCold(size_t physical_offset, size_t length,
                   std::optional<size_t> virtual_offset) const;
+    /// Pages the kernel hands out (cleared or not).
+    void MarkUsed(size_t physical_offset, size_t length);
+
+    /// Pages given back to the kernel: their contents are no longer known to be zero.
+    void MarkDirty(size_t physical_offset, size_t length);
+
+    /// A new kernel on this memory: everything a previous one used may hold data.
+    void ForgetContents();
 
     [[nodiscard]] u8* BackingBasePointer() noexcept {
         return backing_base;
@@ -94,6 +105,12 @@ private:
     u8* backing_base{};
     u8* virtual_base{};
     size_t virtual_base_offset{};
+
+    static constexpr size_t GranuleShift = 16; ///< 64 KiB of backing per state byte
+    static constexpr u8 GranuleUsed = 1 << 0;   ///< Handed out at least once
+    static constexpr u8 GranuleDirty = 1 << 1;  ///< May hold non-zero data
+    std::unique_ptr<std::atomic<u8>[]> granules;
+    size_t num_granules{};
     // Windows requires it for kernels whom lack proper support for some functions!
     bool fallback_buffer{false};
 };