From 51207c73c889dbdc4a586a5da23314f6d33e4eb9 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:26:16 -0400 Subject: Switch 2: Turing macros, native compute, Ampere shader lowering, native tiling From changes.patch: the TU104/GA10x MME interpreter (shared macro context for the process's channels, also on the compute class), native compute with QMD 3.0 dependency scheduling, the Ampere-to-Maxwell shader lowering, native vertex stream sizes, and the console's GOB swizzle for textures, DMA copies and the host unswizzle shaders. Switch 2 channels turn all of it on at init. The lowering is widened for title 4 (all 32 of its programs lower): general TEX (any dimension/LOD/mask, operand split 1+1/2+1/2+2/4+n, any but consistent descriptor buffer), cube/array, SULD channel masks, 128-bit STG, indexed IPA, VOTE, SHFL with any immediate or predicate output, R2P/P2R byte select, uniform ULOP3/ULEA/UBREV/UFLO.SH/UISETP signed/UMOV URZ, IMNMX cbuf, IADD3 and FFMA uniform operands, FMUL /4 /8 x8, and BSSY/BSYNC/BREAK/WARPSYNC as no-ops. The capture stops at unmapped memory. EDEN_NX2_SHADER_DUMP dumps every native program; tools in C:\dev\nx2work test the lowering offline. --- src/video_core/buffer_cache/buffer_cache.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'src/video_core/buffer_cache') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index f30c85911f..126e73c9b6 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -9,6 +9,7 @@ #include #include #include +#include #include "common/range_sets.inc" #include "video_core/buffer_cache/buffer_cache_base.h" @@ -1372,7 +1373,12 @@ void BufferCache

::UpdateVertexBuffer(u32 index) { const auto& array = maxwell3d->regs.vertex_streams[index]; const auto& limit = maxwell3d->regs.vertex_stream_limits[index]; const GPUVAddr gpu_addr_begin = array.Address(); - const GPUVAddr gpu_addr_end = limit.Address() + 1; + const bool native = maxwell3d->GetNativeMacroEngine().native_turing; + const u64 native_size = native ? maxwell3d->regs.NativeVertexStreamSize(index) : 0; + if (native && array.enable != 0 && native_size > (std::numeric_limits::max)()) { + throw std::runtime_error("Native vertex stream exceeds supported 32-bit buffer size"); + } + const GPUVAddr gpu_addr_end = native ? gpu_addr_begin + native_size : limit.Address() + 1; const std::optional device_addr = gpu_memory->GpuToCpuAddress(gpu_addr_begin); const u32 address_size = static_cast(gpu_addr_end - gpu_addr_begin); u32 size = address_size; // TODO: Analyze stride and number of vertices -- cgit v1.2.3 From 315391f3fd9809e8a89e48f05bed887ef9780f9a Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:22:59 -0400 Subject: video_core: 64 GiB device address space Switch 2 titles map most of their memory twice plus GPU-only regions; title 1 ran out of the 16 GiB SMMU space. The buffer cache page table and memory tracker cover the wider space. --- src/video_core/buffer_cache/buffer_cache_base.h | 3 ++- src/video_core/buffer_cache/memory_tracker_base.h | 2 +- src/video_core/host1x/gpu_device_memory_manager.h | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) (limited to 'src/video_core/buffer_cache') diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 26bf3dd9ed..e4aa5896e5 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -544,7 +544,8 @@ private: bool immediately_free = false; #endif - std::array> CACHING_PAGEBITS)> page_table; + // Covers the device address space (MaxwellDeviceTraits::device_virtual_bits). + std::array> CACHING_PAGEBITS)> page_table; Common::ScratchBuffer tmp_buffer; }; diff --git a/src/video_core/buffer_cache/memory_tracker_base.h b/src/video_core/buffer_cache/memory_tracker_base.h index b8547f3149..8936da023f 100644 --- a/src/video_core/buffer_cache/memory_tracker_base.h +++ b/src/video_core/buffer_cache/memory_tracker_base.h @@ -22,7 +22,7 @@ namespace VideoCommon { template class MemoryTrackerBase { - static constexpr size_t MAX_CPU_PAGE_BITS = 34; + static constexpr size_t MAX_CPU_PAGE_BITS = 36; // device addresses (MaxwellDeviceTraits) static constexpr size_t HIGHER_PAGE_BITS = 22; static constexpr size_t HIGHER_PAGE_SIZE = 1ULL << HIGHER_PAGE_BITS; static constexpr size_t HIGHER_PAGE_MASK = HIGHER_PAGE_SIZE - 1ULL; diff --git a/src/video_core/host1x/gpu_device_memory_manager.h b/src/video_core/host1x/gpu_device_memory_manager.h index a9f249991f..1b4f57fe24 100644 --- a/src/video_core/host1x/gpu_device_memory_manager.h +++ b/src/video_core/host1x/gpu_device_memory_manager.h @@ -14,7 +14,9 @@ namespace Tegra { struct MaxwellDeviceMethods; struct MaxwellDeviceTraits { - static constexpr size_t device_virtual_bits = 34; + // 64 GiB: Switch 2 titles map most of their memory twice plus GPU-only regions (title 1 + // ran out of 16 GiB). + static constexpr size_t device_virtual_bits = 36; using DeviceInterface = typename VideoCore::RasterizerInterface; using DeviceMethods = MaxwellDeviceMethods; }; -- cgit v1.2.3 From 55d315c644fdf36405daaf71f75942553a267a6e Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:08:43 -0400 Subject: ASTC: re-compress to BC3 except huge arrays, which stay RGBA8 and trimmed Title 1 has ~700 ASTC 8x8 textures (~280 MB as stored); decoded to RGBA8 they took ~4.5 GB, pushing Eden past the 8 GB of an RTX 4070 Laptop (8.4-9.8 GB in use) and the texture cache into permanent aggressive collection. With astc_recompression=Bc3 (CPU, asynchronous) they fit (4.6-5.6 GB). The two 8192x8192x68 terrain arrays would still not fit as BC3, so ASTC 2D arrays over 1 GiB as RGBA8 always use the GPU decoder with trimmed levels (RGBA8 host image, UNORM/sRGB views), and the decoder pass now exists whenever the GPU lacks native ASTC. Also logs every 600 frames: device memory against the collector thresholds, the largest images, totals by format, and buffer cache totals. Title screen 65-150 s: 10.1 -> 16.0 fps. --- src/video_core/buffer_cache/buffer_cache.h | 16 +++++++ .../renderer_vulkan/vk_texture_cache.cpp | 51 ++++++++++++++++++++-- src/video_core/texture_cache/texture_cache.h | 40 +++++++++++++++++ 3 files changed, 104 insertions(+), 3 deletions(-) (limited to 'src/video_core/buffer_cache') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 126e73c9b6..4df855c0fe 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include "common/range_sets.inc" #include "video_core/buffer_cache/buffer_cache_base.h" @@ -79,6 +81,20 @@ void BufferCache

::TickFrame() { return; } runtime.TickFrame(slot_buffers); + if (static u64 frames{}; ++frames % 600 == 0) { + u64 total{}; + size_t count{}; + std::vector> largest; + for (const auto [buffer_id, buffer] : slot_buffers) { + total += buffer->SizeBytes(); + ++count; + largest.emplace_back(buffer->SizeBytes(), buffer->CpuAddr()); + } + std::partial_sort(largest.begin(), largest.begin() + std::min(6, largest.size()), + largest.end(), std::greater<>{}); + LOG_INFO(HW_GPU, "Buffer cache: {} buffers {} MiB; largest {}", count, total >> 20, + fmt::join(largest.begin(), largest.begin() + std::min(6, largest.size()), ", ")); + } // Calculate hits and shots and move hit bits to the right const u32 hits = std::reduce(channel_state->uniform_cache_hits.begin(), diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index d3f255f05d..12d985fb12 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -148,10 +148,41 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { return usage; } +// An ASTC 2D array whose RGBA8 decode exceeds 1 GiB (title 1's 8192x8192x68 terrain textures). +// These stay on the GPU decoder as RGBA8 with trimmed levels (TrimmedHostInfo) even when other +// ASTC images are re-compressed to BC1/BC3 on the CPU: re-compressing them would not fit either. +[[nodiscard]] bool IsHugeAstcArray(const Device& device, const ImageInfo& info) { + if (!IsPixelFormatASTC(info.format) || device.IsOptimalAstcSupported() || + info.type != ImageType::e2D || info.size.depth != 1) { + return false; + } + u64 bytes = 0; + for (s32 level = 0; level < info.resources.levels; ++level) { + bytes += u64{(std::max)(info.size.width >> level, 1U)} * + (std::max)(info.size.height >> level, 1U) * 4; + } + return bytes * info.resources.layers > (u64{1} << 30); +} + +// The host format of a huge ASTC array: the GPU decoder's RGBA8 (sRGB kept for sampling). +[[nodiscard]] std::optional HugeAstcHostFormat(const Device& device, const ImageInfo& info) { + if (!IsHugeAstcArray(device, info)) { + return std::nullopt; + } + return VideoCore::Surface::IsPixelFormatSRGB(info.format) ? VK_FORMAT_A8B8G8R8_SRGB_PACK32 + : VK_FORMAT_A8B8G8R8_UNORM_PACK32; +} + +constexpr std::array HugeAstcViewFormats{VK_FORMAT_A8B8G8R8_UNORM_PACK32, + VK_FORMAT_A8B8G8R8_SRGB_PACK32}; + [[nodiscard]] bool WillUseAcceleratedAstcDecode(const Device& device, const ImageInfo& info) { if (!IsPixelFormatASTC(info.format) || device.IsOptimalAstcSupported()) { return false; } + if (IsHugeAstcArray(device, info)) { + return true; + } if (Settings::values.accelerate_astc.GetValue() != Settings::AstcDecodeMode::Gpu) { return false; } @@ -990,7 +1021,9 @@ TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& sched : device{device_}, scheduler{scheduler_}, memory_allocator{memory_allocator_}, staging_buffer_pool{staging_buffer_pool_}, blit_image_helper{blit_image_helper_}, render_pass_cache{render_pass_cache_}, resolution{Settings::values.resolution_info} { - if (Settings::values.accelerate_astc.GetValue() == Settings::AstcDecodeMode::Gpu) { + // Huge ASTC arrays always decode on the GPU (IsHugeAstcArray), whatever the decode setting. + if (Settings::values.accelerate_astc.GetValue() == Settings::AstcDecodeMode::Gpu || + !device.IsOptimalAstcSupported()) { astc_decoder_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool, compute_pass_descriptor_queue, memory_allocator); } @@ -1926,7 +1959,10 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu runtime{&runtime_}, original_image(MakeImage(runtime_.device, runtime_.memory_allocator, TrimmedHostInfo(runtime_.device, info, host_level_offset), - runtime->ViewFormats(info.format))), + IsHugeAstcArray(runtime_.device, info) + ? std::span{HugeAstcViewFormats} + : runtime->ViewFormats(info.format), + HugeAstcHostFormat(runtime_.device, info))), aspect_mask(ImageAspectMask(info.format)) { if (original_image.IsUnbacked()) { flags |= VideoCommon::ImageFlagBits::Unbacked; @@ -1934,7 +1970,11 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu if (host_level_offset != 0) { flags |= VideoCommon::ImageFlagBits::TrimmedLevels; } - if (IsPixelFormatASTC(info.format) && !runtime->device.IsOptimalAstcSupported()) { + if (IsHugeAstcArray(runtime->device, info)) { + flags |= VideoCommon::ImageFlagBits::AcceleratedUpload; + flags |= VideoCommon::ImageFlagBits::Converted; + flags |= VideoCommon::ImageFlagBits::CostlyLoad; + } else if (IsPixelFormatASTC(info.format) && !runtime->device.IsOptimalAstcSupported()) { switch (Settings::values.accelerate_astc.GetValue()) { case Settings::AstcDecodeMode::Gpu: if (WillUseAcceleratedAstcDecode(runtime->device, info)) { @@ -2580,6 +2620,11 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI } } auto format_info = MaxwellToVK::SurfaceFormat(*device, FormatType::Optimal, true, format); + if (IsHugeAstcArray(*device, image.info)) { + format_info.format = VideoCore::Surface::IsPixelFormatSRGB(format) + ? VK_FORMAT_A8B8G8R8_SRGB_PACK32 + : VK_FORMAT_A8B8G8R8_UNORM_PACK32; + } if (device->ApiVersion() >= VK_API_VERSION_1_3) { const VkFormatProperties3 properties3 = device->GetPhysical().GetFormatProperties3(format_info.format); diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index e1cb493697..11de37eb6b 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -6,7 +6,10 @@ #pragma once +#include #include +#include +#include #include #include #include "common/container/unordered_map.h" @@ -173,6 +176,43 @@ void TextureCache

::TickFrame() { if (runtime.CanReportMemoryUsage()) { total_used_memory = runtime.GetDeviceMemoryUsage(); } + // Every 600 frames: device memory in use against the collector's thresholds, and what the + // images themselves account for (VRAM pressure makes the collector thrash). + if (static u64 frames{}; ++frames % 600 == 0) { + u64 image_bytes{}; + size_t images{}; + std::vector> largest; + std::map> by_format; + for (const auto [image_id, image] : slot_images) { + const u64 bytes = GetScaledImageSizeBytes(*image); + image_bytes += bytes; + ++images; + largest.emplace_back(bytes, image); + auto& format = by_format[static_cast(image->info.format)]; + format.first += bytes; + ++format.second; + } + LOG_INFO(HW_GPU, "Texture cache: device memory {} MiB (expected {} MiB, critical {} MiB), {} images {} MiB", + total_used_memory >> 20, expected_memory >> 20, critical_memory >> 20, images, + image_bytes >> 20); + std::partial_sort(largest.begin(), largest.begin() + std::min(12, largest.size()), + largest.end(), [](const auto& a, const auto& b) { return a.first > b.first; }); + for (size_t i = 0; i < std::min(12, largest.size()); ++i) { + const Image& image = *largest[i].second; + LOG_INFO(HW_GPU, " {} MiB: {:#x} {}x{}x{} layers {} levels {} format {} type {} flags {:#x}", + largest[i].first >> 20, image.gpu_addr, image.info.size.width, image.info.size.height, + image.info.size.depth, image.info.resources.layers, image.info.resources.levels, + static_cast(image.info.format), static_cast(image.info.type), + static_cast(image.flags)); + } + std::vector> formats; + for (const auto& [format, total] : by_format) formats.emplace_back(total.first, format); + std::sort(formats.rbegin(), formats.rend()); + for (size_t i = 0; i < std::min(8, formats.size()); ++i) { + LOG_INFO(HW_GPU, " format {}: {} MiB in {} images", formats[i].second, formats[i].first >> 20, + by_format[formats[i].second].second); + } + } if (total_used_memory > minimum_memory) { RunGarbageCollector(); } -- cgit v1.2.3 From c01459347405591446d50858d7b075d617a58be9 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:26:52 -0400 Subject: video_core: cheaper small GPU reads, buffer usage tracking and compute uniforms - MemoryManager::ReadBlockUnsafe reads within one 4 KiB page through a single translation (as Read does) instead of the page walk: texture and sampler descriptors, several per draw (2.6% -> 0.1% of title 1's GPU thread). - UsageTracker skips ranges it tracked just before (until Reset): draws bind the same multi-megabyte storage buffers again and again, and each Track wrote a word per 4 KiB (2.0% -> 0.7%). - Small compute uniform buffers the GPU has not written are streamed, as graphics ones are; synchronizing them re-protected pages an inline upload had just unprotected, two VirtualProtect calls per dispatch (1.9% -> 0.7%, page protection 3.2% -> 2.3%). --- src/video_core/buffer_cache/buffer_cache.h | 9 ++++++++- src/video_core/buffer_cache/usage_tracker.h | 17 +++++++++++++++++ src/video_core/memory_manager.cpp | 10 ++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) (limited to 'src/video_core/buffer_cache') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 4df855c0fe..ad2e200438 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -1211,8 +1211,15 @@ void BufferCache

::BindHostComputeUniformBuffers() { return alignment > 1 && (offset % alignment) != 0; } }(); + // Small buffers the GPU has not written are streamed, as graphics ones are: synchronizing + // them made every dispatch re-protect pages an inline upload had just unprotected (two + // VirtualProtect calls per dispatch, ~4,000 dispatches a second in title 1). + const bool use_fast_buffer = + needs_alignment_stream || + (has_host_buffer && size <= channel_state->uniform_buffer_skip_cache_size && + !memory_tracker.IsRegionGpuModified(binding.device_addr, size)); if constexpr (!IS_OPENGL) { - if (needs_alignment_stream) { + if (use_fast_buffer) { const std::span span = runtime.BindMappedUniformBuffer(0, binding_index, size); device_memory.ReadBlockUnsafe(binding.device_addr, span.data(), size); diff --git a/src/video_core/buffer_cache/usage_tracker.h b/src/video_core/buffer_cache/usage_tracker.h index 6f84e463e3..3e2da8bc0b 100644 --- a/src/video_core/buffer_cache/usage_tracker.h +++ b/src/video_core/buffer_cache/usage_tracker.h @@ -6,6 +6,10 @@ #pragma once +#include +#include +#include + #include "common/alignment.h" #include "common/common_types.h" @@ -24,9 +28,19 @@ public: void Reset() noexcept { std::ranges::fill(pages, 0ULL); + recent.fill({~u64{0}, 0}); } void Track(u64 offset, u64 size) noexcept { + // Tracking is idempotent until Reset: skip ranges tracked just before. Draws bind the + // same (often multi-megabyte) storage buffers again and again, and each Track wrote a + // word per 4 KiB of the range. + for (const auto& [recent_offset, recent_size] : recent) { + if (recent_offset == offset && recent_size == size) { + return; + } + } + recent[recent_cursor++ % recent.size()] = {offset, size}; const size_t page = offset >> BUFFER_PAGE_SHIFT; const size_t page_end = (offset + size) >> BUFFER_PAGE_SHIFT; if (page_end < page || page_end >= pages.size()) { @@ -83,6 +97,9 @@ private: private: std::vector pages; + /// Ranges tracked last, (offset, size). + std::array, 4> recent{{{~u64{0}, 0}, {~u64{0}, 0}, {~u64{0}, 0}, {~u64{0}, 0}}}; + size_t recent_cursor{}; }; } // namespace VideoCommon diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 9a9fdab7bc..0eae8fac24 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -410,6 +410,16 @@ void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::siz } void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer, const std::size_t size) const { + // Small reads within one 4 KiB page (texture and sampler descriptors, several per draw): one + // translation and a copy, as Read does, instead of the page walk (~3% of the GPU thread + // in title 1). Anything else, unmapped or sparse pages included, takes the full path. + constexpr std::size_t SmallPage = 0x1000; + if (size <= SmallPage && (gpu_src_addr & (SmallPage - 1)) + size <= SmallPage) { + if (const u8* const pointer = GetPointer(gpu_src_addr)) { + std::memcpy(dest_buffer, pointer, size); + return; + } + } ReadBlockImpl(gpu_src_addr, dest_buffer, size, VideoCommon::CacheType::None, true); } -- cgit v1.2.3 From e08b40252790eef96337409d7dd606aa7bfa8428 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:51:42 -0400 Subject: video_core: less waiting on the GPU caches' locks - The texture and buffer caches' mutexes spin briefly before sleeping (Common::SpinRecursiveMutex): the GPU thread, the fencing thread and CPU write traps take them tens of thousands of times a second for microseconds, and a sleeping waiter's wake-up took longer than that. Title 1's GPU thread slept 7.7% of its time there, now 3.7%. - Fences commit and pop texture and buffer downloads separately, and only when there are any (one fence per compute dispatch in Switch 2 titles): the fencing thread no longer holds both caches while it copies buffer downloads. - Inline uploads and pre-draw invalidation take the caches one at a time again: holding one while std::lock waited for the other waited longer. --- src/common/CMakeLists.txt | 1 + src/common/spin_recursive_mutex.h | 98 +++++++++++++++++++++++ src/video_core/buffer_cache/buffer_cache_base.h | 3 +- src/video_core/fence_manager.h | 49 +++++++++--- src/video_core/renderer_opengl/gl_rasterizer.cpp | 2 +- src/video_core/renderer_vulkan/vk_rasterizer.cpp | 14 +++- src/video_core/texture_cache/texture_cache.h | 5 ++ src/video_core/texture_cache/texture_cache_base.h | 6 +- 8 files changed, 161 insertions(+), 17 deletions(-) create mode 100644 src/common/spin_recursive_mutex.h (limited to 'src/video_core/buffer_cache') diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index aae4afd180..d3e7b03cd2 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -96,6 +96,7 @@ add_library( ${CMAKE_CURRENT_BINARY_DIR}/scm_rev.cpp scm_rev.h scope_exit.h + spin_recursive_mutex.h scratch_buffer.h settings.cpp settings.h diff --git a/src/common/spin_recursive_mutex.h b/src/common/spin_recursive_mutex.h new file mode 100644 index 0000000000..291dd4e41e --- /dev/null +++ b/src/common/spin_recursive_mutex.h @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include + +#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) +#include +#endif + +namespace Common { + +/// A recursive mutex that spins briefly before sleeping. The GPU caches' locks are taken tens of +/// thousands of times a second by the GPU thread, the fencing thread and CPU write traps, and +/// held for microseconds: std::recursive_mutex put a waiting thread to sleep at once, and its +/// wake-up took longer than the critical section (title 1's GPU thread spent ~8% asleep there). +class SpinRecursiveMutex { +public: + void lock() { + const std::thread::id self = std::this_thread::get_id(); + if (owner.load(std::memory_order_relaxed) == self) { + ++depth; + return; + } + std::uint32_t expected = 0; + if (!state.compare_exchange_strong(expected, 1, std::memory_order_acquire, + std::memory_order_relaxed)) { + LockContended(); + } + owner.store(self, std::memory_order_relaxed); + depth = 1; + } + + bool try_lock() { + const std::thread::id self = std::this_thread::get_id(); + if (owner.load(std::memory_order_relaxed) == self) { + ++depth; + return true; + } + std::uint32_t expected = 0; + if (!state.compare_exchange_strong(expected, 1, std::memory_order_acquire, + std::memory_order_relaxed)) { + return false; + } + owner.store(self, std::memory_order_relaxed); + depth = 1; + return true; + } + + void unlock() { + if (--depth != 0) { + return; + } + owner.store(std::thread::id{}, std::memory_order_relaxed); + if (state.exchange(0, std::memory_order_release) == 2) { + state.notify_one(); + } + } + +private: + // About 10 microseconds of spinning before sleeping. + static constexpr int SpinCount = 1024; + + static void Pause() { +#if defined(_M_X64) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) + _mm_pause(); +#elif defined(__aarch64__) + asm volatile("yield"); +#endif + } + + void LockContended() { + for (int spin = 0; spin < SpinCount; ++spin) { + Pause(); + if (state.load(std::memory_order_relaxed) == 0) { + std::uint32_t expected = 0; + if (state.compare_exchange_weak(expected, 1, std::memory_order_acquire, + std::memory_order_relaxed)) { + return; + } + } + } + // 2: held with (possible) sleepers; taken this way it stays marked, so its unlock wakes + // the next sleeper. + while (state.exchange(2, std::memory_order_acquire) != 0) { + state.wait(2, std::memory_order_relaxed); + } + } + + std::atomic state{0}; ///< 0 free, 1 held, 2 held with sleepers + std::atomic owner{}; + std::uint32_t depth{}; ///< Recursion depth, touched only by the owner +}; + +} // namespace Common diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index e4aa5896e5..57011752cc 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -21,6 +21,7 @@ #include #include "common/common_types.h" +#include "common/spin_recursive_mutex.h" #include "common/div_ceil.h" #include "common/literals.h" #include "common/lru_cache.h" @@ -339,7 +340,7 @@ public: } while (channel_state->has_deleted_buffers); } - std::recursive_mutex mutex; + Common::SpinRecursiveMutex mutex; Runtime& runtime; bool any_buffer_uploaded = false; diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h index 65d8f24aae..eb72529da0 100644 --- a/src/video_core/fence_manager.h +++ b/src/video_core/fence_manager.h @@ -81,7 +81,7 @@ public: if constexpr (!can_async_check) { TryReleasePendingFences(); } - CommitAsyncFlushes(); + const u8 committed_flushes = CommitAsyncFlushes(); TFence new_fence = CreateFence(!should_flush); if constexpr (can_async_check) { guard.lock(); @@ -94,6 +94,7 @@ public: pending_operations.emplace_back(std::move(uncommitted_operations)); uncommitted_operations.clear(); // moved-from: make it empty explicitly } + pending_flushes.push_back(committed_flushes); QueueFence(new_fence); if (!delay_fence) { func(); @@ -186,7 +187,8 @@ private: return; } } - PopAsyncFlushes(); + PopAsyncFlushes(pending_flushes.front()); + pending_flushes.pop_front(); auto operations = std::move(pending_operations.front()); pending_operations.pop_front(); for (auto& operation : operations) { @@ -207,6 +209,7 @@ private: TFence current_fence; std::deque> current_operations; + u8 current_flushes{}; while (!stop_token.stop_requested()) { { std::unique_lock lock(guard); @@ -216,13 +219,15 @@ private: } current_fence = std::move(fences.front()); current_operations = std::move(pending_operations.front()); + current_flushes = pending_flushes.front(); fences.pop(); pending_operations.pop_front(); + pending_flushes.pop_front(); } if (!current_fence->IsStubbed()) { WaitFence(current_fence); } - PopAsyncFlushes(); + PopAsyncFlushes(current_flushes); for (auto& operation : current_operations) { operation(); } @@ -245,27 +250,51 @@ private: query_cache.HasUncommittedFlushes(); } - void PopAsyncFlushes() { - { - std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; + static constexpr u8 TextureFlushes = 1; + static constexpr u8 BufferFlushes = 2; + + /// committed: what CommitAsyncFlushes returned for this fence. + void PopAsyncFlushes(u8 committed) { + if (committed & TextureFlushes) { + std::scoped_lock lock{texture_cache.mutex}; texture_cache.PopAsyncFlushes(); + } + if (committed & BufferFlushes) { + std::scoped_lock lock{buffer_cache.mutex}; buffer_cache.PopAsyncFlushes(); } query_cache.PopAsyncFlushes(); } - void CommitAsyncFlushes() { + /// Returns which caches committed downloads for this fence (TextureFlushes, BufferFlushes). + /// Switch 2 titles signal a fence per compute dispatch (thousands a second), mostly with + /// nothing or only buffer data to download: the fencing thread then takes only the lock of + /// the cache with work, instead of holding both while it copies (the GPU thread's draws and + /// uploads waited for it). + u8 CommitAsyncFlushes() { + u8 committed = 0; { - std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; - texture_cache.CommitAsyncFlushes(); - buffer_cache.CommitAsyncFlushes(); + std::scoped_lock lock{texture_cache.mutex}; + if (texture_cache.HasAsyncFlushWork()) { + texture_cache.CommitAsyncFlushes(); + committed |= TextureFlushes; + } + } + { + std::scoped_lock lock{buffer_cache.mutex}; + if (buffer_cache.HasUncommittedFlushes()) { + buffer_cache.CommitAsyncFlushes(); + committed |= BufferFlushes; + } } query_cache.CommitAsyncFlushes(); + return committed; } std::queue fences; std::deque> uncommitted_operations; std::deque>> pending_operations; + std::deque pending_flushes; ///< Per fence: the caches that committed downloads std::mutex guard; std::mutex ring_guard; diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 1d77d28c46..4e0972d1d2 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -716,7 +716,7 @@ void RasterizerOpenGL::AccelerateInlineToMemory(GPUVAddr address, size_t copy_si } gpu_memory->WriteBlockUnsafe(address, memory.data(), copy_size); { - std::unique_lock lock{buffer_cache.mutex}; + std::unique_lock lock{buffer_cache.mutex}; if (!buffer_cache.InlineMemory(*cpu_addr, copy_size, memory)) { buffer_cache.WriteMemory(*cpu_addr, copy_size); } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 886c5dbfe4..429b2d63ef 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -1036,11 +1036,13 @@ void RasterizerVulkan::InvalidateRegion(DAddr addr, u64 size, VideoCommon::Cache void RasterizerVulkan::InnerInvalidation(std::span> sequences) { { - // Both caches under one lock: this runs before most draws (constant buffer updates). - std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; + std::scoped_lock lock{texture_cache.mutex}; for (const auto& [addr, size] : sequences) { texture_cache.WriteMemory(addr, size); } + } + { + std::scoped_lock lock{buffer_cache.mutex}; for (const auto& [addr, size] : sequences) { buffer_cache.WriteMemory(addr, size); } @@ -1248,12 +1250,16 @@ void RasterizerVulkan::AccelerateInlineToMemory(GPUVAddr address, size_t copy_si return; } gpu_memory->WriteBlockUnsafe(address, memory.data(), copy_size); + // One cache at a time: holding both while waiting for either (std::lock) waited longer, the + // fencing thread and CPU write traps holding one of them most of the time. { - // Both caches under one lock (title 1 makes thousands of inline uploads a second). - std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; + std::scoped_lock lock{buffer_cache.mutex}; if (!buffer_cache.InlineMemory(*cpu_addr, copy_size, memory)) { buffer_cache.WriteMemory(*cpu_addr, copy_size); } + } + { + std::scoped_lock lock{texture_cache.mutex}; texture_cache.WriteMemory(*cpu_addr, copy_size); } pipeline_cache.InvalidateRegion(*cpu_addr, copy_size); diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 13dfbba20e..fb8adef161 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -989,6 +989,11 @@ bool TextureCache

::HasUncommittedFlushes() const noexcept { return !uncommitted_downloads.empty(); } +template +bool TextureCache

::HasAsyncFlushWork() const noexcept { + return !uncommitted_downloads.empty() || !uncommitted_async_buffers.empty(); +} + template bool TextureCache

::ShouldWaitAsyncFlushes() const noexcept { return !committed_downloads.empty() && !committed_downloads.front().empty(); diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 3c96c91a93..11ac8a2421 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -23,6 +23,7 @@ #include #include "common/common_types.h" +#include "common/spin_recursive_mutex.h" #include "common/hash.h" #include "common/literals.h" #include "common/lru_cache.h" @@ -244,6 +245,9 @@ public: /// Return true when there are uncommitted images to be downloaded [[nodiscard]] bool HasUncommittedFlushes() const noexcept; + /// Whether CommitAsyncFlushes would commit anything (downloads or their buffers). + [[nodiscard]] bool HasAsyncFlushWork() const noexcept; + /// Return true when the caller should wait for async downloads [[nodiscard]] bool ShouldWaitAsyncFlushes() const noexcept; @@ -276,7 +280,7 @@ public: /// Prepare an image to be used void PrepareImage(ImageId image_id, bool is_modification, bool invalidate); - std::recursive_mutex mutex; + Common::SpinRecursiveMutex mutex; private: /// Iterate over all page indices in a range -- cgit v1.2.3