core: use the 12 GiB layout and auto-stub IPC for Switch 2 titles

Switch 2 titles size their heap from the total memory and stop at the
first unimplemented command, so they only booted with the 12 GiB layout
and EDEN_NX2_AUTO_STUB set by hand. Detect them from main.npdm before the
kernel starts and apply the layout as a per-game value (the configured
layout comes back once the game is closed), and auto-stub unknown
commands from Switch 2 processes.

4 files changed, 35 insertions, 2 deletions

diff --git a/src/core/core.cpp b/src/core/core.cpp
index eee91ef4fc..f3d224b1ce 100644
--- a/src/core/core.cpp
+++ b/src/core/core.cpp
@@ -24,6 +24,7 @@
 #include "core/device_memory.h"
 #include "core/file_sys/fs_filesystem.h"
 #include "core/file_sys/patch_manager.h"
+#include "core/file_sys/program_metadata.h"
 #include "core/file_sys/registered_cache.h"
 #include "core/file_sys/romfs_factory.h"
 #include "core/file_sys/savedata_factory.h"
@@ -109,6 +110,19 @@ FileSys::VirtualFile GetGameFileFromPath(const FileSys::VirtualFilesystem& vfs,
     return vfs->OpenFile(path, FileSys::OpenMode::Read);
 }
 
+namespace {
+
+// Switch 2 titles boot as a loose ExeFS, with main.npdm next to main.
+bool IsSwitch2Program(const FileSys::VirtualFile& file) {
+    const auto dir = file != nullptr ? file->GetContainingDirectory() : nullptr;
+    const auto npdm = dir != nullptr ? dir->GetFile("main.npdm") : nullptr;
+    FileSys::ProgramMetadata metadata;
+    return npdm != nullptr && metadata.Load(npdm) == Loader::ResultStatus::Success &&
+           metadata.IsSwitch2();
+}
+
+} // namespace
+
 struct System::Impl {
     explicit Impl(System& system)
         : kernel{system}, fs_controller{system}, hid_core{kernel}, cpu_manager{system},
@@ -299,6 +313,16 @@ struct System::Impl {
             fs_controller.InitTempStorage();
         }
 
+        // Switch 2 titles size their heap from the total memory and need the 12 GiB layout. Set it
+        // as a per-game value, so RestoreGlobalState brings back the configured layout afterwards.
+        if (virtual_filesystem != nullptr &&
+            Settings::values.memory_layout_mode.GetValue() != Settings::MemoryLayout::Memory_12Gb &&
+            IsSwitch2Program(GetGameFileFromPath(virtual_filesystem, filepath))) {
+            LOG_INFO(Core, "Switch 2 program: using the 12 GiB memory layout");
+            Settings::values.memory_layout_mode.SetGlobal(false);
+            Settings::values.memory_layout_mode.SetValue(Settings::MemoryLayout::Memory_12Gb);
+        }
+
         InitializeKernel(system);
 
         if (params.applet_type == Service::AM::AppletType::Application) {
diff --git a/src/core/hle/kernel/k_process.cpp b/src/core/hle/kernel/k_process.cpp
index 4aac818b77..c3a3b73559 100644
--- a/src/core/hle/kernel/k_process.cpp
+++ b/src/core/hle/kernel/k_process.cpp
@@ -1265,6 +1265,7 @@ Result KProcess::LoadFromMetadata(KernelCore& kernel, const FileSys::ProgramMeta
     // its heap as total memory minus ~6.3 GiB), so count the alias region as well, bounded by the
     // pool through the resource limit.
     if (metadata.IsSwitch2()) {
+        m_is_switch2 = true;
         m_max_process_memory = m_page_table.GetHeapRegionSize() + m_page_table.GetAliasRegionSize();
     }
 
diff --git a/src/core/hle/kernel/k_process.h b/src/core/hle/kernel/k_process.h
index b5693751c3..d20711c222 100644
--- a/src/core/hle/kernel/k_process.h
+++ b/src/core/hle/kernel/k_process.h
@@ -135,6 +135,7 @@ private:
     bool m_is_signaled : 1 = false;
     bool m_is_initialized : 1 = false;
     bool m_is_application : 1 = false;
+    bool m_is_switch2 : 1 = false;
     bool m_is_default_application_system_resource : 1 = false;
     bool m_is_suspended : 1 = false;
     bool m_is_immortal : 1 = false;
@@ -248,6 +249,10 @@ public:
         return m_is_application;
     }
 
+    bool IsSwitch2() const {
+        return m_is_switch2;
+    }
+
     bool IsDefaultApplicationSystemResource() const {
         return m_is_default_application_system_resource;
     }
diff --git a/src/core/hle/service/service.cpp b/src/core/hle/service/service.cpp
index 5a779af3ca..1490af0328 100644
--- a/src/core/hle/service/service.cpp
+++ b/src/core/hle/service/service.cpp
@@ -80,10 +80,13 @@ void ServiceFrameworkBase::ReportUnimplementedFunction(HLERequestContext& ctx,
 
     system.GetReporter().SaveUnimplementedFunctionReport(ctx, ctx.GetCommand(), function_name, service_name);
     UNIMPLEMENTED_MSG("Unknown / unimplemented {}", fmt::to_string(buf));
-    // EDEN_NX2_AUTO_STUB=1 auto-stubs every unknown command without touching the config.
+    // EDEN_NX2_AUTO_STUB=1 auto-stubs every unknown command without touching the config. Switch 2
+    // programs are always auto-stubbed: they still call many commands nobody has implemented.
     static const bool env_auto_stub = std::getenv("EDEN_NX2_AUTO_STUB") != nullptr;
+    const auto* const caller = ctx.GetThread().GetOwnerProcess();
     if (Settings::values.use_auto_stub || env_auto_stub ||
-        std::string_view{service_name}.starts_with(Nx2StubPrefix)) {
+        std::string_view{service_name}.starts_with(Nx2StubPrefix) ||
+        (caller != nullptr && caller->IsSwitch2())) {
         LOG_WARNING(Service, "Using auto stub fallback!");
         IPC::ResponseBuilder rb{ctx, 2};
         rb.Push(ResultSuccess);