1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
|
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#pragma once
#include <atomic>
#include <cstdint>
#include <thread>
#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)
#include <immintrin.h>
#endif
namespace Common {
/// A recursive mutex that spins briefly before sleeping. The GPU caches' locks are taken tens of
/// thousands of times a second by the GPU thread, the fencing thread and CPU write traps, and
/// held for microseconds: std::recursive_mutex put a waiting thread to sleep at once, and its
/// wake-up took longer than the critical section (title 1's GPU thread spent ~8% asleep there).
class SpinRecursiveMutex {
public:
void lock() {
const std::thread::id self = std::this_thread::get_id();
if (owner.load(std::memory_order_relaxed) == self) {
++depth;
return;
}
std::uint32_t expected = 0;
if (!state.compare_exchange_strong(expected, 1, std::memory_order_acquire,
std::memory_order_relaxed)) {
LockContended();
}
owner.store(self, std::memory_order_relaxed);
depth = 1;
}
bool try_lock() {
const std::thread::id self = std::this_thread::get_id();
if (owner.load(std::memory_order_relaxed) == self) {
++depth;
return true;
}
std::uint32_t expected = 0;
if (!state.compare_exchange_strong(expected, 1, std::memory_order_acquire,
std::memory_order_relaxed)) {
return false;
}
owner.store(self, std::memory_order_relaxed);
depth = 1;
return true;
}
void unlock() {
if (--depth != 0) {
return;
}
owner.store(std::thread::id{}, std::memory_order_relaxed);
if (state.exchange(0, std::memory_order_release) == 2) {
state.notify_one();
}
}
private:
// About 10 microseconds of spinning before sleeping.
static constexpr int SpinCount = 1024;
static void Pause() {
#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__)
_mm_pause();
#elif defined(__aarch64__)
asm volatile("yield");
#endif
}
void LockContended() {
for (int spin = 0; spin < SpinCount; ++spin) {
Pause();
if (state.load(std::memory_order_relaxed) == 0) {
std::uint32_t expected = 0;
if (state.compare_exchange_weak(expected, 1, std::memory_order_acquire,
std::memory_order_relaxed)) {
return;
}
}
}
// 2: held with (possible) sleepers; taken this way it stays marked, so its unlock wakes
// the next sleeper.
while (state.exchange(2, std::memory_order_acquire) != 0) {
state.wait(2, std::memory_order_relaxed);
}
}
std::atomic<std::uint32_t> state{0}; ///< 0 free, 1 held, 2 held with sleepers
std::atomic<std::thread::id> owner{};
std::uint32_t depth{}; ///< Recursion depth, touched only by the owner
};
} // namespace Common
|
