nx2: give sparse regions memory only where the GPU writes

nx2:gpu 214 regions with the sparse flag (word 7 bit 0x200) are Mario
Kart World's terrain texture arrays and layer pools, 170 MB to 5.8 GB,
mostly empty. Backed whole they ran the graphics pool dry (26 "could not
allocate" per session, later regions without memory), and backing a
block at a time made the texture cache rebuild the 1.45 GB arrays each
time a block was mapped.

A sparse region now gets a device (SMMU) range of its own when it is
created, mapped for the GPU at once, so its GPU mapping never changes.
Before a GPU-side write lands in it (WriteBlock, Write<T>), the 4 MiB
blocks it touches get memory, cleared, behind their device pages.
Unbacked device pages read zero and drop writes instead of crashing, and
the unmapped-device-access errors are logged 16 times, not once per page.
nx2:gpu 215 releases the device range and the blocks after the GPU is
done with them. The log reports GPU-only memory in use and its peak.

Mario Kart World, 180 s: no allocation failures; ~2.8 GB of terrain
blocks get memory instead of 6.5 GB of whole regions, next to 3 GB of
ordinary GPU-only regions. Terrain renders as before.

1 files changed, 22 insertions, 8 deletions

diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp
index 2dd8b04656..9111ba10d2 100644
--- a/src/video_core/memory_manager.cpp
+++ b/src/video_core/memory_manager.cpp
@@ -14,6 +14,7 @@
 #include "core/hle/kernel/k_process.h"
 #include "memory_manager.h"
 #include "video_core/guest_memory.h"
+#include "video_core/nx2.h"
 #include "video_core/host1x/host1x.h"
 #include "video_core/invalidation_accumulator.h"
 #include "video_core/memory_manager.h"
@@ -119,7 +120,6 @@ GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr de
     if (entry_type == EntryType::Mapped) {
         page_table.ReserveRange(gpu_addr, size);
     }
-    for (u64 offset{}; offset < size; offset += page_size) {
     // The caches hear about each run of pages whose entry changed in one call, not page by page
     // (a 1.45 GB Switch 2 region made ~370,000 texture cache scans).
     GPUVAddr modified_start{};
@@ -130,6 +130,7 @@ GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr de
             modified_size = 0;
         }
     };
+    for (u64 offset{}; offset < size; offset += page_size) {
         const GPUVAddr current_gpu_addr = gpu_addr + offset;
         [[maybe_unused]] const auto current_entry_type = GetEntry(current_gpu_addr, false);
         SetEntry(current_gpu_addr, entry_type, false);
@@ -149,14 +150,13 @@ GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr de
         }
         remaining_size -= page_size;
     }
-    kind_map.Map(gpu_addr, gpu_addr + size, kind);
     flush_modified();
+    kind_map.Map(gpu_addr, gpu_addr + size, kind);
     return gpu_addr;
 }
 
 GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr dev_addr, size_t size, PTEKind kind, MemoryManager::EntryType entry_type) {
     [[maybe_unused]] u64 remaining_size{size};
-    for (u64 offset{}; offset < size; offset += big_page_size) {
     GPUVAddr modified_start{};
     u64 modified_size{};
     const auto flush_modified = [&] {
@@ -165,6 +165,7 @@ GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr
             modified_size = 0;
         }
     };
+    for (u64 offset{}; offset < size; offset += big_page_size) {
         const GPUVAddr current_gpu_addr = gpu_addr + offset;
         [[maybe_unused]] const auto current_entry_type = GetEntry(current_gpu_addr, true);
         SetEntry(current_gpu_addr, entry_type, true);
@@ -201,8 +202,8 @@ GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr
         }
         remaining_size -= big_page_size;
     }
-    {
     flush_modified();
+    {
         std::unique_lock<std::mutex> lock(guard);
         kind_map.Map(gpu_addr, gpu_addr + size, kind);
     }
@@ -300,6 +301,12 @@ void MemoryManager::Write(GPUVAddr addr, T data) {
         std::memcpy(page_pointer, &data, sizeof(T));
         return;
     }
+    // A Switch 2 sparse block nobody wrote yet gets its memory now.
+    Tegra::Nx2BackSparseWrite(addr, sizeof(T));
+    if (auto page_pointer{GetPointer(addr)}; page_pointer) {
+        std::memcpy(page_pointer, &data, sizeof(T));
+        return;
+    }
 
     ASSERT(false);
 }
@@ -411,8 +418,12 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std:
         if (!unsafe) {
             rasterizer->FlushRegion(dev_addr_base, copy_amount, which);
         }
-        u8* physical = memory.GetPointer<u8>(dev_addr_base);
-        std::memcpy(dest_buffer, physical, copy_amount);
+        // A device page without memory (a sparse block nobody wrote yet) reads zero.
+        if (const u8* physical = memory.GetPointer<u8>(dev_addr_base)) [[likely]] {
+            std::memcpy(dest_buffer, physical, copy_amount);
+        } else {
+            std::memset(dest_buffer, 0, copy_amount);
+        }
         dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
     };
     auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
@@ -454,6 +465,8 @@ void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer, co
 }
 
 void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, [[maybe_unused]] VideoCommon::CacheType which, bool unsafe) {
+    // Give Switch 2 sparse pages the write lands on their memory first (otherwise it is dropped).
+    Tegra::Nx2BackSparseWrite(gpu_dest_addr, size);
     auto just_advance = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
         src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
     };
@@ -462,8 +475,9 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe
         if (!unsafe) {
             rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which);
         }
-        u8* physical = memory.GetPointer<u8>(dev_addr_base);
-        std::memcpy(physical, src_buffer, copy_amount);
+        if (u8* physical = memory.GetPointer<u8>(dev_addr_base)) [[likely]] {
+            std::memcpy(physical, src_buffer, copy_amount);
+        }
         src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
     };
     auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {