kernel: spin briefly before sleeping on a contended KSpinLock

The scheduler lock is taken by every emulated core thousands of times a
frame (title 1's job system: WaitForAddress / SignalToAddress / yield)
and held for microseconds. As a std::mutex, a contended Lock put the
host thread to sleep at once. Now a three-state futex lock: one atomic
op uncontended, ~128 pause spins when contended, then
std::atomic::wait. Time blocked in Lock fell ~20-25% on title 1's cores.

2 files changed, 53 insertions, 5 deletions

diff --git a/src/core/hle/kernel/k_spin_lock.cpp b/src/core/hle/kernel/k_spin_lock.cpp
index 8525320374..107d6f1aa3 100644
--- a/src/core/hle/kernel/k_spin_lock.cpp
+++ b/src/core/hle/kernel/k_spin_lock.cpp
@@ -3,18 +3,60 @@
 
 #include "core/hle/kernel/k_spin_lock.h"
 
+#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)
+#include <immintrin.h>
+#endif
+
 namespace Kernel {
 
+namespace {
+
+// Spins of about 30-140 cycles each before a contended Lock sleeps (a few microseconds).
+constexpr int SpinCount = 128;
+
+void Pause() {
+#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)
+    _mm_pause();
+#elif defined(__aarch64__)
+    asm volatile("yield");
+#endif
+}
+
+} // Anonymous namespace
+
 void KSpinLock::Lock() {
-    m_lock.lock();
+    u32 expected = 0;
+    if (m_state.compare_exchange_strong(expected, 1, std::memory_order_acquire,
+                                        std::memory_order_relaxed)) {
+        return;
+    }
+    for (int spin = 0; spin < SpinCount; ++spin) {
+        Pause();
+        if (m_state.load(std::memory_order_relaxed) == 0) {
+            expected = 0;
+            if (m_state.compare_exchange_weak(expected, 1, std::memory_order_acquire,
+                                              std::memory_order_relaxed)) {
+                return;
+            }
+        }
+    }
+    // Mark the lock contended and sleep until it is released; taken this way it stays marked,
+    // so its Unlock wakes the next sleeper.
+    while (m_state.exchange(2, std::memory_order_acquire) != 0) {
+        m_state.wait(2, std::memory_order_relaxed);
+    }
 }
 
 void KSpinLock::Unlock() {
-    m_lock.unlock();
+    if (m_state.exchange(0, std::memory_order_release) == 2) {
+        m_state.notify_one();
+    }
 }
 
 bool KSpinLock::TryLock() {
-    return m_lock.try_lock();
+    u32 expected = 0;
+    return m_state.compare_exchange_strong(expected, 1, std::memory_order_acquire,
+                                           std::memory_order_relaxed);
 }
 
 } // namespace Kernel
diff --git a/src/core/hle/kernel/k_spin_lock.h b/src/core/hle/kernel/k_spin_lock.h
index 094a1e6be0..f7bc8b4d05 100644
--- a/src/core/hle/kernel/k_spin_lock.h
+++ b/src/core/hle/kernel/k_spin_lock.h
@@ -3,9 +3,11 @@
 
 #pragma once
 
-#include <mutex>
+#include <atomic>
+#include <mutex> // Kernel headers that use std::mutex get it through this one
 
 #include "common/common_funcs.h"
+#include "common/common_types.h"
 #include "core/hle/kernel/k_scoped_lock.h"
 
 namespace Kernel {
@@ -22,7 +24,11 @@ public:
     bool TryLock();
 
 private:
-    std::mutex m_lock;
+    // 0 free, 1 held, 2 held with (possible) sleepers. The scheduler lock is taken by every
+    // core thousands of times a frame and held for microseconds: a contended Lock spins briefly
+    // before sleeping, instead of putting the host thread to sleep at once and paying its
+    // wake-up latency (std::mutex did).
+    std::atomic<u32> m_state{0};
 };
 
 // TODO(bunnei): Alias for now, in case we want to implement these accurately in the future.