From 97ce27ab35153884d2fd05c278433928e429cd91 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Fri, 30 Oct 2026 04:39:44 -0400 Subject: texture_cache: re-decode only the written layers of large ASTC arrays Any write into an image marked all of it CPU-modified, so each terrain tile Mario Kart World streams into its 8192x8192x68 ASTC arrays re-read and re-decoded the whole 1.45 GB array: a 2 GiB staging buffer and a full GPU wait each time, 15-16 GB of uploads per 600 frames. Large layered images on the accelerated ASTC path (64 MiB and up) now record the layers written since their last upload and stay tracked; a refresh reads and decodes only those layers, one dispatch per layer and level into a single-layer storage view. The texture cache's 600-frame report counts uploads from guest memory, their size and how many went by layer. Upload scratch buffers grown past 64 MiB for one huge image (341 MiB for the terrain layer pools) are freed at the next frame instead of kept for the session. Mario Kart World, 180 s: after loading, uploads drop from 15-16 GB to 0.2-0.5 GB per 600 frames, the largest from 1453 MiB to 2-15 MiB. --- src/video_core/texture_cache/texture_cache.h | 108 ++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 2 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 62cd523073..d582f520f7 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -198,6 +198,13 @@ void TextureCache

::ApplyThresholdOverrides(bool log) { template void TextureCache

::TickFrame() { + // Upload buffers grown for one huge image (title 1's 341 MiB terrain layer pools) are freed + // rather than kept for the rest of the session. + for (auto* const scratch : {&swizzle_data_buffer, &unswizzle_data_buffer}) { + if (scratch->capacity() > LargeScratchCapacity) { + scratch->release(); + } + } if constexpr (HAS_DEVICE_MEMORY_INFO) { // Switch 2 titles (native tiling) stream far more texture data than Eden's thresholds // were tuned for: title 1's attract loop alone keeps ~5.3 GB resident, so the collector @@ -280,6 +287,9 @@ void TextureCache

::TickFrame() { images_created, images_deleted, gc_deletes + gc_costly_deletes, gc_costly_deletes, gc_download_deletes, async_decodes_queued, astc_opaque_images, astc_alpha_images); + LOG_INFO(HW_GPU, "Texture cache last 600 frames: {} uploads from guest memory, {} MiB (largest {} MiB, {} by layer)", + images_uploaded, uploaded_bytes >> 20, largest_upload >> 20, layer_uploads); + images_uploaded = uploaded_bytes = largest_upload = layer_uploads = 0; astc_opaque_images = astc_alpha_images = 0; std::vector> shapes; for (const auto& [key, value] : created_shapes) shapes.emplace_back(value.first, key); @@ -706,7 +716,11 @@ FramebufferId TextureCache

::GetFramebufferId(const RenderTargets& key) { template void TextureCache

::WriteMemory(DAddr cpu_addr, size_t size) { - ForEachImageInRegion(cpu_addr, size, [this](ImageId image_id, Image& image) { + ForEachImageInRegion(cpu_addr, size, [this, cpu_addr, size](ImageId image_id, Image& image) { + if (UploadsByLayer(image)) { + MarkLayersWritten(image, cpu_addr, size); + return; + } if (True(image.flags & ImageFlagBits::CpuModified)) { return; } @@ -812,6 +826,7 @@ void TextureCache

::UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t siz UntrackImage(image, id); } } + image.dirty_layers.clear(); // all of it if (True(image.flags & ImageFlagBits::Remapped)) { continue; } @@ -1280,7 +1295,16 @@ void TextureCache

::RefreshContents(Image& image, ImageId image_id) { image.flags &= ~ImageFlagBits::CpuModified; - TrackImage(image, image_id); + // Images uploaded by layer stay tracked while some of their layers are dirty. + if (False(image.flags & ImageFlagBits::Tracked)) { + TrackImage(image, image_id); + } + + if (!image.dirty_layers.empty() && UploadWrittenLayers(image)) { + runtime.InsertUploadMemoryBarrier(); + return; + } + image.dirty_layers.clear(); if (True(image.flags & ImageFlagBits::Unbacked)) { // Nothing to upload into (and staging it could take tens of GB). @@ -1311,11 +1335,91 @@ void TextureCache

::RefreshContents(Image& image, ImageId image_id) { QueueAsyncUnswizzle(image, image_id); return; } + ++images_uploaded; + uploaded_bytes += image.guest_size_bytes; + largest_upload = (std::max)(largest_upload, image.guest_size_bytes); auto staging = runtime.UploadStagingBuffer(MapSizeBytes(image)); UploadImageContents(image, staging); runtime.InsertUploadMemoryBarrier(); } +template +bool TextureCache

::UploadsByLayer(const ImageBase& image) const noexcept { + // Title 1's terrain arrays (8192x8192x68 ASTC, 1.45 GB each) get a few layers streamed in at a + // time; re-decoding all of them for each took a 2 GiB staging buffer and stalled the GPU. + if constexpr (requires { &Runtime::AccelerateImageUploadLayers; }) { + return True(image.flags & ImageFlagBits::AcceleratedUpload) && + image.info.type == ImageType::e2D && image.info.resources.layers > 1 && + image.guest_size_bytes >= LayerUploadMinSize && + False(image.flags & (ImageFlagBits::Unbacked | ImageFlagBits::Sparse | + ImageFlagBits::Remapped)); + } else { + return false; + } +} + +template +void TextureCache

::MarkLayersWritten(Image& image, DAddr cpu_addr, size_t size) { + if (True(image.flags & ImageFlagBits::CpuModified) && image.dirty_layers.empty()) { + return; // all of it is dirty already + } + const u32 layers = static_cast(image.info.resources.layers); + if (False(image.flags & ImageFlagBits::CpuModified)) { + image.flags |= ImageFlagBits::CpuModified; + image.dirty_layers.assign(Common::DivCeil(layers, 64U), 0); + } + const DAddr begin = (std::max)(cpu_addr, image.cpu_addr) - image.cpu_addr; + const DAddr end = (std::min)(cpu_addr + size, image.cpu_addr + image.guest_size_bytes) - + image.cpu_addr; + const u32 stride = image.info.layer_stride; + if (end <= begin || stride == 0) { + return; + } + const u32 last = (std::min)(static_cast((end - 1) / stride), layers - 1); + for (u32 layer = static_cast(begin / stride); layer <= last; ++layer) { + image.dirty_layers[layer / 64] |= u64{1} << (layer % 64); + } +} + +template +bool TextureCache

::UploadWrittenLayers(Image& image) { + if constexpr (requires { &Runtime::AccelerateImageUploadLayers; }) { + const u32 num_layers = static_cast(image.info.resources.layers); + std::vector layers; + for (u32 layer = 0; layer < num_layers; ++layer) { + if ((image.dirty_layers[layer / 64] >> (layer % 64)) & 1) { + layers.push_back(layer); + } + } + image.dirty_layers.clear(); + if (layers.empty() || layers.size() == num_layers) { + return false; + } + // Each layer's mip chain from the first level the host keeps on (as UploadImageContents). + auto uploads = FullUploadSwizzles(image.info); + const size_t level_offset = uploads[image.host_level_offset].buffer_offset; + const size_t tail = image.info.layer_stride - level_offset; + auto staging = runtime.UploadStagingBuffer(tail * layers.size()); + for (size_t i = 0; i < layers.size(); ++i) { + gpu_memory->ReadBlock(image.gpu_addr + u64{layers[i]} * image.info.layer_stride + + level_offset, + staging.mapped_span.data() + i * tail, tail, + VideoCommon::CacheType::NoTextureCache); + } + uploads.erase(uploads.begin(), uploads.begin() + image.host_level_offset); + for (auto& upload : uploads) { + upload.buffer_offset -= level_offset; + } + runtime.AccelerateImageUploadLayers(image, staging, uploads, layers); + ++images_uploaded; + uploaded_bytes += tail * layers.size(); + ++layer_uploads; + return true; + } else { + return false; + } +} + template template void TextureCache

::UploadImageContents(Image& image, StagingBuffer& staging) { -- cgit v1.2.3 From 940d44ca25628288f1b5f64c25c022b3a4ae5ce2 Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:16:34 -0400 Subject: texture_cache: give images over unwritten sparse blocks no device memory Mario Kart World creates nine terrain layer pools in sparse regions (four R8 16384x16384, four RG16 8192x8192, one R16 8192x8192: 2.9 GB of device memory). In 180 s of title and attract demo not one of their blocks is written and nothing renders into them, so they hold zeros; on the console only the few tiles the game maps have memory. nx2:gpu now publishes which 4 MiB blocks of each sparse region have been written (nx2.h, a leaf lock the texture cache may take). An image of 32 MiB or more lying in one sparse region with no written block is created as a sparse resident image with nothing bound: reads return zero, as an unmapped tile does. The first write into its range, or the first time the GPU renders into it (that range then counts as written), marks it remapped, so a backed image replaces it at its next use. Mario Kart World, 180 s: the collector deletes 0-80 images per 600 frames instead of thousands, frames over 100 ms per 600 presents drop from 1-9 to 0-1, peak working set 14.7 GB. One render target in a small sparse region gets replaced once (logged). --- src/core/hle/service/nx2_gsv.h | 3 + src/video_core/nx2.h | 76 ++++++++++++++++++++++ .../renderer_vulkan/vk_texture_cache.cpp | 30 ++++++++- src/video_core/texture_cache/image_base.h | 3 + src/video_core/texture_cache/texture_cache.h | 14 ++++ .../vulkan_common/vulkan_memory_allocator.cpp | 29 ++++++--- .../vulkan_common/vulkan_memory_allocator.h | 4 ++ 7 files changed, 147 insertions(+), 12 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/core/hle/service/nx2_gsv.h b/src/core/hle/service/nx2_gsv.h index f3cc87ac4d..9d00b81368 100644 --- a/src/core/hle/service/nx2_gsv.h +++ b/src/core/hle/service/nx2_gsv.h @@ -549,6 +549,7 @@ private: // BackSparseWrite gives a block memory before the GPU writes into it. m_sparse_regions[start] = {size, std::vector(Common::DivCeil(size, SparseBlockSize))}; Nvidia::Devices::Nx2GpuMemoryRegistry::Get().CreateSparse(start, size); + Tegra::Nx2SparseRegionCreated(start, size, SparseBlockSize); Tegra::Nx2AddSparseSpan(start, size); LOG_INFO(Service, "nx2:gpu 214 sparse region [{:#x}, {:#x}] backed as the GPU writes it", start, last); @@ -580,6 +581,7 @@ private: FreeReleasedRegions(); if (last > first) { if (m_sparse_regions.erase(first) != 0) { + Tegra::Nx2SparseRegionDestroyed(first); Nvidia::Devices::Nx2GpuMemoryRegistry::Get().DestroySparse(first, last - first + 1); } else { Nvidia::Devices::Nx2GpuMemoryRegistry::Get().Unmap(first, last - first + 1); @@ -723,6 +725,7 @@ private: region.blocks[block] = BlockBacked; Nvidia::Devices::Nx2GpuMemoryRegistry::Get().BackSparse(block_address, *address, block_size); + Tegra::Nx2SparseBlockWritten(start, block); m_sparse_backed_bytes += block_size; if (++m_sparse_blocks == 1 || (m_sparse_blocks % 64) == 0) { LOG_INFO(Service, "nx2:gpu {} sparse blocks backed ({} MiB); {} MiB of GPU-only " diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index 58c9b17bcf..1b565e73d4 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -93,6 +94,81 @@ inline void Nx2BackSparseWrite(u64 gpu_address, u64 size) { } } +// Which blocks of each sparse region have been written (got memory), for the texture cache: an +// image over blocks nobody wrote is all zeros and needs no device memory. Its own lock, taken last: +// the texture cache asks with its lock held, and nx2:gpu updates it holding none of the GPU's. +struct Nx2SparseRegionState { + u64 size; + u64 block_size; + std::vector written; +}; +inline std::mutex g_nx2_sparse_state_mutex; +inline std::map g_nx2_sparse_state; + +inline void Nx2SparseRegionCreated(u64 gpu_address, u64 size, u64 block_size) { + std::scoped_lock lk{g_nx2_sparse_state_mutex}; + g_nx2_sparse_state[gpu_address] = {size, block_size, + std::vector((size + block_size - 1) / block_size)}; +} + +inline void Nx2SparseRegionDestroyed(u64 gpu_address) { + std::scoped_lock lk{g_nx2_sparse_state_mutex}; + g_nx2_sparse_state.erase(gpu_address); +} + +inline void Nx2SparseBlockWritten(u64 region_address, u64 block) { + std::scoped_lock lk{g_nx2_sparse_state_mutex}; + if (const auto it = g_nx2_sparse_state.find(region_address); it != g_nx2_sparse_state.end()) { + it->second.written[block] = true; + } +} + +/// The GPU renders into [gpu_address, +size): count its blocks as written, so images made there +/// later get memory (their contents live in the host image, not in guest memory). +inline void Nx2SparseMarkWritten(u64 gpu_address, u64 size) { + if (size == 0 || gpu_address + size <= g_nx2_sparse_low.load(std::memory_order_relaxed) || + gpu_address >= g_nx2_sparse_high.load(std::memory_order_relaxed)) [[likely]] { + return; + } + std::scoped_lock lk{g_nx2_sparse_state_mutex}; + for (auto& [start, region] : g_nx2_sparse_state) { + const u64 end = start + region.size; + if (gpu_address >= end || gpu_address + size <= start) { + continue; + } + const u64 first = ((std::max)(gpu_address, start) - start) / region.block_size; + const u64 last = ((std::min)(gpu_address + size, end) - 1 - start) / region.block_size; + for (u64 block = first; block <= last; ++block) { + region.written[block] = true; + } + } +} + +/// True when [gpu_address, +size) lies in one sparse region and none of it has been written. +inline bool Nx2SparseUnwritten(u64 gpu_address, u64 size) { + if (size == 0 || gpu_address + size <= g_nx2_sparse_low.load(std::memory_order_relaxed) || + gpu_address >= g_nx2_sparse_high.load(std::memory_order_relaxed)) [[likely]] { + return false; + } + std::scoped_lock lk{g_nx2_sparse_state_mutex}; + auto it = g_nx2_sparse_state.upper_bound(gpu_address); + if (it == g_nx2_sparse_state.begin()) { + return false; + } + --it; + const auto& [start, region] = *it; + if (gpu_address + size > start + region.size) { + return false; + } + const u64 last = (gpu_address + size - 1 - start) / region.block_size; + for (u64 block = (gpu_address - start) / region.block_size; block <= last; ++block) { + if (region.written[block]) { + return false; + } + } + return true; +} + /// GPU VA in the first page that mirrors a release at `address`, or 0. inline u64 Nx2SemaphoreMirror(u64 address) { for (const auto& region : g_nx2_semaphore_regions) { diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 27f0e58321..08a471cfd6 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -20,6 +20,7 @@ #include "common/settings.h" #include "video_core/renderer_vulkan/vk_texture_cache.h" +#include "video_core/nx2.h" #include "video_core/gpu_logging/gpu_logging.h" #include "video_core/engines/fermi_2d.h" @@ -165,6 +166,16 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { return bytes * info.resources.layers > (u64{1} << 30); } +// A large image over Switch 2 sparse blocks nobody has written (title 1's terrain layer pools, 2.9 +// GB of R8/RG16/R16 that stay empty in the attract demo) is all zeros: it gets no device memory +// until a write makes the texture cache replace it. +[[nodiscard]] bool IsUnwrittenSparse(const ImageInfo& info, GPUVAddr gpu_addr) { + constexpr u32 MinSize = 32U << 20; + const u32 size = VideoCommon::CalculateGuestSizeInBytes(info); + return size >= MinSize && info.type != ImageType::e1D && info.type != ImageType::Buffer && + info.type != ImageType::Linear && Tegra::Nx2SparseUnwritten(gpu_addr, size); +} + // The host format of a huge ASTC array: the GPU decoder's RGBA8 (sRGB kept for sampling). [[nodiscard]] std::optional HugeAstcHostFormat(const Device& device, const ImageInfo& info) { if (!IsHugeAstcArray(device, info)) { @@ -310,7 +321,8 @@ constexpr std::array OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM [[nodiscard]] vk::Image MakeImage(const Device& device, const MemoryAllocator& allocator, const ImageInfo& info, std::span view_formats, - std::optional format_override = {}) { + std::optional format_override = {}, + bool unbacked = false) { if (info.type == ImageType::Buffer) { return vk::Image{}; } @@ -338,6 +350,11 @@ constexpr std::array OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM image_ci.pNext = &image_format_list; } } + if (unbacked) { + if (vk::Image image = allocator.CreateUnbackedImage(image_ci)) { + return image; + } + } return allocator.CreateImage(image_ci); } @@ -2019,10 +2036,19 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu : IsOpaqueBc1Astc(runtime_.device, info) ? std::span{OpaqueAstcViewFormats} : runtime->ViewFormats(info.format), - AstcHostFormat(runtime_.device, info))), + AstcHostFormat(runtime_.device, info), + IsUnwrittenSparse(info, gpu_addr_))), aspect_mask(ImageAspectMask(info.format)) { if (original_image.IsUnbacked()) { flags |= VideoCommon::ImageFlagBits::Unbacked; + // Not too big to back (see MemoryAllocator::CreateImage): unbacked because nothing was + // written there yet, so the first write gets it replaced. + const ImageInfo host_info = TrimmedHostInfo(runtime_.device, info, host_level_offset); + const u64 texels = u64{host_info.size.width} * host_info.size.height * + host_info.size.depth * static_cast(host_info.resources.layers); + if (texels <= (u64{1} << 28)) { + flags |= VideoCommon::ImageFlagBits::UnwrittenSparse; + } } if (host_level_offset != 0) { flags |= VideoCommon::ImageFlagBits::TrimmedLevels; diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h index 20d3883130..cfb839d17e 100644 --- a/src/video_core/texture_cache/image_base.h +++ b/src/video_core/texture_cache/image_base.h @@ -50,6 +50,9 @@ enum class ImageFlagBits : u32 { /// in VRAM); uploads skip the rest, downloads are never done. TrimmedLevels = 1 << 19, ZeroFilled = 1 << 20, ///< Cleared to zero blocks before its first asynchronous decode. + /// Over Switch 2 sparse blocks nobody wrote: all zeros, so the host image has no memory + /// (Unbacked) until the first write, which makes a backed image replace it. + UnwrittenSparse = 1 << 21, }; DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits) diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index d582f520f7..168df50c8b 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -28,6 +28,7 @@ #include "video_core/texture_cache/samples_helper.h" #include "video_core/texture_cache/texture_cache_base.h" #include "video_core/texture_cache/util.h" +#include "video_core/nx2.h" #include "video_core/nx2_profile.h" #include "video_core/textures/decoders.h" @@ -717,6 +718,11 @@ FramebufferId TextureCache

::GetFramebufferId(const RenderTargets& key) { template void TextureCache

::WriteMemory(DAddr cpu_addr, size_t size) { ForEachImageInRegion(cpu_addr, size, [this, cpu_addr, size](ImageId image_id, Image& image) { + if (True(image.flags & ImageFlagBits::UnwrittenSparse)) { + // Written at last: a backed image replaces it where it is used next. + image.flags |= ImageFlagBits::CpuModified | ImageFlagBits::Remapped; + return; + } if (UploadsByLayer(image)) { MarkLayersWritten(image, cpu_addr, size); return; @@ -2918,6 +2924,14 @@ void TextureCache

::PrepareImage(ImageId image_id, bool is_modification, bool } } if (is_modification) { + if (True(image.flags & ImageFlagBits::UnwrittenSparse) && + False(image.flags & ImageFlagBits::Remapped)) { + // The GPU renders into it: these writes are lost, later uses get a backed image. + LOG_WARNING(HW_GPU, "GPU write into unwritten sparse image {:#x}; replacing it", + image.gpu_addr); + Tegra::Nx2SparseMarkWritten(image.gpu_addr, image.guest_size_bytes); + image.flags |= ImageFlagBits::CpuModified | ImageFlagBits::Remapped; + } MarkModification(image); } lru_cache.Touch(image.lru_index, frame_tick); diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index 8bf692b00e..92f537aa3c 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -248,21 +248,15 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicat } if (result == VK_ERROR_OUT_OF_DEVICE_MEMORY && ci.imageType != VK_IMAGE_TYPE_1D) { // Too big to back at all, e.g. title 1's sparse virtual texture arrays (8192x8192, - // 68 layers, ~23 GB): on the console only resident tiles have memory. Create a sparse - // resident image with nothing bound, so views and copies still work; reads of it return - // zero and writes are dropped, instead of aborting. - VkImageCreateInfo sparse_ci = ci; - sparse_ci.flags |= VK_IMAGE_CREATE_SPARSE_BINDING_BIT | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT; - VkImage sparse{}; - if (device.GetDispatchLoader().vkCreateImage(*device.GetLogical(), &sparse_ci, nullptr, - &sparse) == VK_SUCCESS) { + // 68 layers, ~23 GB): on the console only resident tiles have memory. Use an image with + // nothing bound, so views and copies still work, instead of aborting. + if (vk::Image sparse = CreateUnbackedImage(ci)) { LOG_WARNING(Render_Vulkan, "Image {}x{}x{} ({} levels, {} layers, format {}) does not fit; using an " "unbacked sparse image", ci.extent.width, ci.extent.height, ci.extent.depth, ci.mipLevels, ci.arrayLayers, static_cast(ci.format)); - return vk::Image(sparse, ci.usage, *device.GetLogical(), allocator, VK_NULL_HANDLE, - device.GetDispatchLoader()); + return sparse; } } if (result != VK_SUCCESS) { @@ -286,6 +280,21 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicat device.GetDispatchLoader()); } +vk::Image MemoryAllocator::CreateUnbackedImage(const VkImageCreateInfo &ci) const { + if (ci.imageType == VK_IMAGE_TYPE_1D) { + return vk::Image{}; + } + VkImageCreateInfo sparse_ci = ci; + sparse_ci.flags |= VK_IMAGE_CREATE_SPARSE_BINDING_BIT | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT; + VkImage sparse{}; + if (device.GetDispatchLoader().vkCreateImage(*device.GetLogical(), &sparse_ci, nullptr, + &sparse) != VK_SUCCESS) { + return vk::Image{}; + } + return vk::Image(sparse, ci.usage, *device.GetLogical(), allocator, VK_NULL_HANDLE, + device.GetDispatchLoader()); +} + vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const { // MESA will do memcpy() if not marked as host cached, so just force mark it for most buffers auto const anv_flags = (usage == MemoryUsage::Stream diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index fa7d4b41c3..99649b9aef 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -107,6 +107,10 @@ namespace Vulkan { /// images would otherwise leave holes in the shared blocks). vk::Image CreateImage(const VkImageCreateInfo &ci, bool dedicated = false) const; + /// A sparse resident image with nothing bound (reads return zero, writes are dropped), or + /// a null image when the device can't make one. + vk::Image CreateUnbackedImage(const VkImageCreateInfo &ci) const; + vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const; vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage, -- cgit v1.2.3 From d4ee7fcd5129e7ee32ff38a5aef9b449d5a756de Mon Sep 17 00:00:00 2001 From: Takoroka <137927095+Takoroka@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:28:53 -0400 Subject: vulkan: bind memory to sparse terrain arrays layer by layer The written part of Mario Kart World's sparse terrain arrays is small: 13-26% of their blocks. The two 1.45 GB ASTC arrays were backed in full on the host, and the two 5.8 GB BC6H arrays, too large to back, stayed unbacked, so what the game streamed into them never showed. A color array of 32 MiB or more lying in one sparse region, in a format uploaded layer by layer (huge ASTC arrays decoded on the GPU, or formats sampled as they are), is now a sparse resident image (SparseLayers). When it is refreshed, the layers written since (or, for a new image, the layers any of whose blocks was written) get device memory bound with vkQueueBindSparse (under the submit lock, waited on, as the multi-range buffer cache does) and only those layers are uploaded. Unwritten layers read zero. One image binds at most 768 MiB (384 MiB on GPUs with less than 7 GiB); rendering into one binds all its layers. Mario Kart World, 180 s: the ASTC arrays hold 1.4 MiB per written layer, four BC6H layers (341 MiB) now have their data, device memory in use 4.8-5.0 GB instead of 6.0-7.1 GB with the texture cache below its collection threshold, 16.8-17.2 ms average frames, peak working set 13.4 GB. --- src/video_core/nx2.h | 15 ++ .../renderer_vulkan/vk_texture_cache.cpp | 231 ++++++++++++++++++++- src/video_core/renderer_vulkan/vk_texture_cache.h | 65 ++++++ src/video_core/texture_cache/image_base.h | 3 + src/video_core/texture_cache/texture_cache.h | 86 +++++++- src/video_core/texture_cache/texture_cache_base.h | 6 + src/video_core/vulkan_common/vulkan_device.h | 5 + .../vulkan_common/vulkan_memory_allocator.cpp | 29 +++ .../vulkan_common/vulkan_memory_allocator.h | 6 + src/video_core/vulkan_common/vulkan_wrapper.cpp | 11 + src/video_core/vulkan_common/vulkan_wrapper.h | 4 + 11 files changed, 453 insertions(+), 8 deletions(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index 1b565e73d4..1a3bae93ec 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -123,6 +123,21 @@ inline void Nx2SparseBlockWritten(u64 region_address, u64 block) { } } +/// True when [gpu_address, +size) lies in one sparse region. +inline bool Nx2InSparseRegion(u64 gpu_address, u64 size) { + if (size == 0 || gpu_address + size <= g_nx2_sparse_low.load(std::memory_order_relaxed) || + gpu_address >= g_nx2_sparse_high.load(std::memory_order_relaxed)) [[likely]] { + return false; + } + std::scoped_lock lk{g_nx2_sparse_state_mutex}; + auto it = g_nx2_sparse_state.upper_bound(gpu_address); + if (it == g_nx2_sparse_state.begin()) { + return false; + } + --it; + return gpu_address + size <= it->first + it->second.size; +} + /// The GPU renders into [gpu_address, +size): count its blocks as written, so images made there /// later get memory (their contents live in the host image, not in guest memory). inline void Nx2SparseMarkWritten(u64 gpu_address, u64 size) { diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 08a471cfd6..b77f9af5f8 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -16,7 +16,9 @@ #include #include #include +#include "common/alignment.h" #include "common/bit_util.h" +#include "common/div_ceil.h" #include "common/settings.h" #include "video_core/renderer_vulkan/vk_texture_cache.h" @@ -176,6 +178,31 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { info.type != ImageType::Linear && Tegra::Nx2SparseUnwritten(gpu_addr, size); } +// A large color array in a Switch 2 sparse region (title 1's terrain arrays: 13-26% of their +// blocks written) gets memory by layer, as its layers are written: unbacked at first, then +// Image::BindLayers. Only formats uploaded layer by layer: huge ASTC arrays (GPU decode) and +// formats the host samples as they are. +[[nodiscard]] bool UsesSparseLayers(const Device& device, const ImageInfo& info, GPUVAddr gpu_addr) { + if (!device.IsSparseResidencyImage2DSupported() || info.type != ImageType::e2D || + info.resources.layers <= 1 || info.num_samples != 1 || + VideoCore::Surface::GetFormatType(info.format) != VideoCore::Surface::SurfaceType::ColorTexture) { + return false; + } + const bool layer_uploads = + IsHugeAstcArray(device, info) || + (!IsPixelFormatASTC(info.format) && + (!IsPixelFormatBCn(info.format) || device.IsOptimalBcnSupported())); + const u32 size = VideoCommon::CalculateGuestSizeInBytes(info); + return layer_uploads && size >= (32U << 20) && Tegra::Nx2InSparseRegion(gpu_addr, size); +} + +// The most device memory one SparseLayers image binds: 768 MiB, 384 MiB on GPUs with less than +// 7 GiB (title 1's BC6H arrays are 85 MB a layer). +[[nodiscard]] VkDeviceSize SparseLayersCap(const Device& device) { + return device.GetDeviceLocalMemory() < (u64{7} << 30) ? (VkDeviceSize{384} << 20) + : (VkDeviceSize{768} << 20); +} + // The host format of a huge ASTC array: the GPU decoder's RGBA8 (sRGB kept for sampling). [[nodiscard]] std::optional HugeAstcHostFormat(const Device& device, const ImageInfo& info) { if (!IsHugeAstcArray(device, info)) { @@ -2037,16 +2064,19 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu ? std::span{OpaqueAstcViewFormats} : runtime->ViewFormats(info.format), AstcHostFormat(runtime_.device, info), - IsUnwrittenSparse(info, gpu_addr_))), + UsesSparseLayers(runtime_.device, info, gpu_addr_) || + IsUnwrittenSparse(info, gpu_addr_))), aspect_mask(ImageAspectMask(info.format)) { if (original_image.IsUnbacked()) { flags |= VideoCommon::ImageFlagBits::Unbacked; - // Not too big to back (see MemoryAllocator::CreateImage): unbacked because nothing was - // written there yet, so the first write gets it replaced. const ImageInfo host_info = TrimmedHostInfo(runtime_.device, info, host_level_offset); const u64 texels = u64{host_info.size.width} * host_info.size.height * host_info.size.depth * static_cast(host_info.resources.layers); - if (texels <= (u64{1} << 28)) { + if (UsesSparseLayers(runtime_.device, info, gpu_addr_) && InitSparseLayers()) { + flags |= VideoCommon::ImageFlagBits::SparseLayers; + } else if (texels <= (u64{1} << 28)) { + // Not too big to back (see MemoryAllocator::CreateImage): unbacked because nothing + // was written there yet, so the first write gets it replaced. flags |= VideoCommon::ImageFlagBits::UnwrittenSparse; } } @@ -2520,6 +2550,199 @@ VkImageView Image::StorageImageView(s32 level) noexcept { return *view; } +bool Image::InitSparseLayers() { + const Device& device = runtime->device; + const VkImage handle = *original_image; + const auto requirements = device.GetLogical().GetImageSparseMemoryRequirements(handle); + const auto it = std::ranges::find_if(requirements, [](const auto& r) { + return (r.formatProperties.aspectMask & VK_IMAGE_ASPECT_COLOR_BIT) != 0; + }); + if (it == requirements.end()) { + return false; + } + SparseLayerMemory& s = sparse_layers; + s.allocator = &runtime->memory_allocator; + s.requirements = device.GetLogical().GetImageMemoryRequirements(handle); + s.sparse = *it; + u32 offset = host_level_offset; + const ImageInfo host = TrimmedHostInfo(device, info, offset); + s.width = host.size.width; + s.height = host.size.height; + s.levels = static_cast(host.resources.levels); + // Sparse block size in texels (drivers give it in texels or in compressed blocks). + const bool huge_astc = IsHugeAstcArray(device, info); + const u32 block_width = huge_astc ? 1 : VideoCore::Surface::DefaultBlockWidth(info.format); + const u32 block_height = huge_astc ? 1 : VideoCore::Surface::DefaultBlockHeight(info.format); + const u64 block_bytes = huge_astc ? 4 : VideoCore::Surface::BytesPerBlock(info.format); + const VkExtent3D granularity = it->formatProperties.imageGranularity; + s.tile_width = granularity.width; + s.tile_height = granularity.height; + const u64 tile = s.requirements.alignment; + if (u64{granularity.width / block_width} * (granularity.height / block_height) * block_bytes != + tile && + u64{granularity.width} * granularity.height * block_bytes == tile) { + s.tile_width *= block_width; + s.tile_height *= block_height; + } + const u32 tail_lod = (std::min)(it->imageMipTailFirstLod, s.levels); + VkDeviceSize bytes = 0; + for (u32 level = 0; level < tail_lod; ++level) { + const u32 width = (std::max)(s.width >> level, 1U); + const u32 height = (std::max)(s.height >> level, 1U); + bytes += u64{Common::DivCeil(width, s.tile_width)} * Common::DivCeil(height, s.tile_height) * + tile; + } + if (tail_lod < s.levels && + (it->formatProperties.flags & VK_SPARSE_IMAGE_FORMAT_SINGLE_MIPTAIL_BIT) == 0) { + bytes += it->imageMipTailSize; + } + s.layer_bytes = Common::AlignUp(bytes, tile); + s.layer_memory.assign(static_cast(info.resources.layers), VK_NULL_HANDLE); + LOG_INFO(Render_Vulkan, + "Sparse layered image {:#x}: {}x{} {} levels {} layers, {} KiB a layer, block {}x{}", + gpu_addr, s.width, s.height, s.levels, info.resources.layers, s.layer_bytes >> 10, + s.tile_width, s.tile_height); + return true; +} + +std::vector Image::BindLayers(std::span layers) { + SparseLayerMemory& s = sparse_layers; + std::vector bound; + if (s.layer_memory.empty()) { + return bound; + } + const Device& device = runtime->device; + const u64 tile = s.requirements.alignment; + const u32 tail_lod = (std::min)(s.sparse.imageMipTailFirstLod, s.levels); + const bool has_tail = tail_lod < s.levels; + const bool single_tail = + (s.sparse.formatProperties.flags & VK_SPARSE_IMAGE_FORMAT_SINGLE_MIPTAIL_BIT) != 0; + const VkDeviceSize cap = SparseLayersCap(device); + std::vector image_binds; + std::vector opaque_binds; + for (const u32 layer : layers) { + if (s.layer_memory[layer] != VK_NULL_HANDLE) { + bound.push_back(layer); + continue; + } + if (s.bound_bytes + s.layer_bytes > cap) { + if (!std::exchange(s.warned, true)) { + LOG_WARNING(Render_Vulkan, "Sparse layered image {:#x}: {} MiB bound, more layers " + "stay without memory", + gpu_addr, s.bound_bytes >> 20); + } + continue; + } + const VkMemoryRequirements requirements{ + .size = s.layer_bytes, + .alignment = tile, + .memoryTypeBits = s.requirements.memoryTypeBits, + }; + VkDeviceMemory memory{}; + VkDeviceSize offset{}; + const VmaAllocation allocation = s.allocator->AllocateSparseMemory(requirements, memory, offset); + if (allocation == VK_NULL_HANDLE) { + LOG_WARNING(Render_Vulkan, "Sparse layered image {:#x}: out of device memory", gpu_addr); + break; + } + s.layer_memory[layer] = allocation; + s.bound_bytes += s.layer_bytes; + for (u32 level = 0; level < tail_lod; ++level) { + const u32 width = (std::max)(s.width >> level, 1U); + const u32 height = (std::max)(s.height >> level, 1U); + image_binds.push_back(VkSparseImageMemoryBind{ + .subresource{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .mipLevel = level, + .arrayLayer = layer, + }, + .offset{0, 0, 0}, + .extent{width, height, 1}, + .memory = memory, + .memoryOffset = offset, + .flags = 0, + }); + offset += u64{Common::DivCeil(width, s.tile_width)} * + Common::DivCeil(height, s.tile_height) * tile; + } + if (has_tail && !single_tail) { + opaque_binds.push_back(VkSparseMemoryBind{ + .resourceOffset = s.sparse.imageMipTailOffset + layer * s.sparse.imageMipTailStride, + .size = s.sparse.imageMipTailSize, + .memory = memory, + .memoryOffset = offset, + .flags = 0, + }); + } + bound.push_back(layer); + } + if (has_tail && single_tail && s.tail_memory == VK_NULL_HANDLE && !bound.empty()) { + const VkMemoryRequirements requirements{ + .size = Common::AlignUp(s.sparse.imageMipTailSize, tile), + .alignment = tile, + .memoryTypeBits = s.requirements.memoryTypeBits, + }; + VkDeviceMemory memory{}; + VkDeviceSize offset{}; + s.tail_memory = s.allocator->AllocateSparseMemory(requirements, memory, offset); + if (s.tail_memory != VK_NULL_HANDLE) { + opaque_binds.push_back(VkSparseMemoryBind{ + .resourceOffset = s.sparse.imageMipTailOffset, + .size = s.sparse.imageMipTailSize, + .memory = memory, + .memoryOffset = offset, + .flags = 0, + }); + } + } + if (image_binds.empty() && opaque_binds.empty()) { + return bound; + } + const VkSparseImageMemoryBindInfo image_bind_info{ + .image = *original_image, + .bindCount = static_cast(image_binds.size()), + .pBinds = image_binds.data(), + }; + const VkSparseImageOpaqueMemoryBindInfo opaque_bind_info{ + .image = *original_image, + .bindCount = static_cast(opaque_binds.size()), + .pBinds = opaque_binds.data(), + }; + const VkBindSparseInfo bind_info{ + .sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO, + .pNext = nullptr, + .waitSemaphoreCount = 0, + .pWaitSemaphores = nullptr, + .bufferBindCount = 0, + .pBufferBinds = nullptr, + .imageOpaqueBindCount = opaque_binds.empty() ? 0U : 1U, + .pImageOpaqueBinds = opaque_binds.empty() ? nullptr : &opaque_bind_info, + .imageBindCount = image_binds.empty() ? 0U : 1U, + .pImageBinds = image_binds.empty() ? nullptr : &image_bind_info, + .signalSemaphoreCount = 0, + .pSignalSemaphores = nullptr, + }; + // Bound before the command buffer that uploads the layers is submitted (as the multi-range + // buffer cache does). + vk::Fence fence = device.GetLogical().CreateFence(VkFenceCreateInfo{ + .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + }); + VkResult result; + { + std::scoped_lock lock{scheduler->submit_mutex}; + result = device.GetGraphicsQueue().BindSparse(bind_info, *fence); + } + if (result != VK_SUCCESS) { + LOG_ERROR(Render_Vulkan, "Sparse layered image {:#x}: binding failed ({})", gpu_addr, + static_cast(result)); + return {}; + } + fence.Wait(); + return bound; +} + VkImageView Image::StorageImageLayerView(s32 level, u32 layer) { const u32 layers = static_cast(info.resources.layers); if (storage_layer_views.empty()) { diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 59f2936e06..13919f1270 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -385,6 +385,10 @@ public: /// A storage view of one layer of a host level (decoding only some layers of an array). VkImageView StorageImageLayerView(s32 level, u32 layer); + /// SparseLayers images: gives `layers` device memory where they have none yet. Returns the + /// layers that have memory afterwards (a cap per image can leave some out). + std::vector BindLayers(std::span layers); + bool IsRescaled() const noexcept; bool ScaleUp(bool ignore = false); @@ -403,6 +407,67 @@ private: Scheduler* scheduler{}; TextureCacheRuntime* runtime{}; + /// Memory bound to the layers of a SparseLayers image. Declared before the images, so it is + /// released after them. + struct SparseLayerMemory { + SparseLayerMemory() = default; + SparseLayerMemory(const SparseLayerMemory&) = delete; + SparseLayerMemory& operator=(const SparseLayerMemory&) = delete; + SparseLayerMemory(SparseLayerMemory&& other) noexcept { + *this = std::move(other); + } + SparseLayerMemory& operator=(SparseLayerMemory&& other) noexcept { + Release(); + allocator = std::exchange(other.allocator, nullptr); + requirements = other.requirements; + sparse = other.sparse; + width = other.width; + height = other.height; + levels = other.levels; + tile_width = other.tile_width; + tile_height = other.tile_height; + layer_bytes = other.layer_bytes; + layer_memory = std::move(other.layer_memory); + other.layer_memory.clear(); + tail_memory = std::exchange(other.tail_memory, VK_NULL_HANDLE); + bound_bytes = other.bound_bytes; + warned = other.warned; + return *this; + } + ~SparseLayerMemory() { + Release(); + } + void Release() noexcept { + if (allocator == nullptr) { + return; + } + for (const VmaAllocation allocation : layer_memory) { + allocator->FreeSparseMemory(allocation); + } + allocator->FreeSparseMemory(tail_memory); + layer_memory.clear(); + tail_memory = VK_NULL_HANDLE; + } + + const MemoryAllocator* allocator{}; + VkMemoryRequirements requirements{}; + VkSparseImageMemoryRequirements sparse{}; + u32 width{}; ///< Host level 0 + u32 height{}; + u32 levels{}; ///< Host levels + u32 tile_width{}; ///< Sparse block, in texels + u32 tile_height{}; + VkDeviceSize layer_bytes{}; ///< One layer's levels before the mip tail, and its own tail + std::vector layer_memory; ///< Per layer, null until bound + VmaAllocation tail_memory{}; ///< The mip tail all layers share, if one + VkDeviceSize bound_bytes{}; + bool warned{}; + }; + SparseLayerMemory sparse_layers; + + /// Reads the image's sparse requirements; false when it can't be bound by layer. + bool InitSparseLayers(); + vk::Image original_image; vk::Image scaled_image; diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h index cfb839d17e..04254fb417 100644 --- a/src/video_core/texture_cache/image_base.h +++ b/src/video_core/texture_cache/image_base.h @@ -53,6 +53,9 @@ enum class ImageFlagBits : u32 { /// Over Switch 2 sparse blocks nobody wrote: all zeros, so the host image has no memory /// (Unbacked) until the first write, which makes a backed image replace it. UnwrittenSparse = 1 << 21, + /// A large array over Switch 2 sparse blocks: unbacked, each layer gets device memory when + /// something is written to it (Image::BindLayers) and only those layers are uploaded. + SparseLayers = 1 << 22, }; DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits) diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 168df50c8b..3de82d2eb4 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include #include @@ -1306,6 +1307,10 @@ void TextureCache

::RefreshContents(Image& image, ImageId image_id) { TrackImage(image, image_id); } + if (True(image.flags & ImageFlagBits::SparseLayers)) { + RefreshSparseLayers(image); + return; + } if (!image.dirty_layers.empty() && UploadWrittenLayers(image)) { runtime.InsertUploadMemoryBarrier(); return; @@ -1354,6 +1359,9 @@ bool TextureCache

::UploadsByLayer(const ImageBase& image) const noexcept { // Title 1's terrain arrays (8192x8192x68 ASTC, 1.45 GB each) get a few layers streamed in at a // time; re-decoding all of them for each took a 2 GiB staging buffer and stalled the GPU. if constexpr (requires { &Runtime::AccelerateImageUploadLayers; }) { + if (True(image.flags & ImageFlagBits::SparseLayers)) { + return true; + } return True(image.flags & ImageFlagBits::AcceleratedUpload) && image.info.type == ImageType::e2D && image.info.resources.layers > 1 && image.guest_size_bytes >= LayerUploadMinSize && @@ -1401,6 +1409,71 @@ bool TextureCache

::UploadWrittenLayers(Image& image) { if (layers.empty() || layers.size() == num_layers) { return false; } + UploadLayers(image, layers); + return true; + } else { + return false; + } +} + +template +void TextureCache

::RefreshSparseLayers(Image& image) { + if constexpr (requires { &Image::BindLayers; }) { + // The layers written since the last upload, or, for a new image, every layer any of whose + // blocks has been written. + const u32 num_layers = static_cast(image.info.resources.layers); + const u64 stride = image.info.layer_stride; + std::vector layers; + for (u32 layer = 0; layer < num_layers; ++layer) { + const bool written = + image.dirty_layers.empty() + ? !Tegra::Nx2SparseUnwritten(image.gpu_addr + layer * stride, stride) + : ((image.dirty_layers[layer / 64] >> (layer % 64)) & 1) != 0; + if (written) { + layers.push_back(layer); + } + } + image.dirty_layers.clear(); + const std::vector bound = layers.empty() ? std::vector{} : image.BindLayers(layers); + if (bound.empty()) { + runtime.TransitionImageLayout(image); + return; + } + UploadLayers(image, bound); + runtime.InsertUploadMemoryBarrier(); + } +} + +template +void TextureCache

::UploadLayers(Image& image, std::span layers) { + if constexpr (requires { &Runtime::AccelerateImageUploadLayers; }) { + if (False(image.flags & ImageFlagBits::AcceleratedUpload)) { + // Plain formats: each layer unswizzled on its own into its layer of the image. + ImageInfo layer_info = image.info; + layer_info.resources.layers = 1; + const size_t layer_size = CalculateUnswizzledSizeBytes(layer_info); + auto staging = runtime.UploadStagingBuffer(layer_size * layers.size()); + std::vector copies; + for (size_t i = 0; i < layers.size(); ++i) { + const GPUVAddr layer_addr = image.gpu_addr + u64{layers[i]} * image.info.layer_stride; + Tegra::Memory::GpuGuestMemory + swizzle_data(*gpu_memory, layer_addr, image.info.layer_stride, + &swizzle_data_buffer); + auto layer_copies = + UnswizzleImage(*gpu_memory, layer_addr, layer_info, swizzle_data, + staging.mapped_span.subspan(i * layer_size, layer_size)); + for (BufferImageCopy copy : layer_copies) { + copy.buffer_offset += i * layer_size; + copy.image_subresource.base_layer = static_cast(layers[i]); + copies.push_back(copy); + } + } + image.UploadMemory(staging, copies); + ++images_uploaded; + uploaded_bytes += u64{image.info.layer_stride} * layers.size(); + ++layer_uploads; + return; + } // Each layer's mip chain from the first level the host keeps on (as UploadImageContents). auto uploads = FullUploadSwizzles(image.info); const size_t level_offset = uploads[image.host_level_offset].buffer_offset; @@ -1420,9 +1493,6 @@ bool TextureCache

::UploadWrittenLayers(Image& image) { ++images_uploaded; uploaded_bytes += tail * layers.size(); ++layer_uploads; - return true; - } else { - return false; } } @@ -2524,7 +2594,8 @@ ImageViewId TextureCache

::FindOrEmplaceImageView(ImageId image_id, const Imag if (const ImageViewId image_view_id = image.FindView(info); image_view_id) { return image_view_id; } - if (True(image.flags & ImageFlagBits::Unbacked)) { + if (True(image.flags & ImageFlagBits::Unbacked) && + False(image.flags & ImageFlagBits::SparseLayers)) { // How the game uses an image too large to back (e.g. a sparse virtual texture). LOG_INFO(HW_GPU, "View of unbacked image {:#x}: type {} format {} levels {}+{} layers {}+{}", image.gpu_addr, static_cast(info.type), static_cast(info.format), @@ -2924,6 +2995,13 @@ void TextureCache

::PrepareImage(ImageId image_id, bool is_modification, bool } } if (is_modification) { + if constexpr (requires { &Image::BindLayers; }) { + if (True(image.flags & ImageFlagBits::SparseLayers)) { + std::vector layers(static_cast(image.info.resources.layers)); + std::iota(layers.begin(), layers.end(), 0U); + image.BindLayers(layers); + } + } if (True(image.flags & ImageFlagBits::UnwrittenSparse) && False(image.flags & ImageFlagBits::Remapped)) { // The GPU renders into it: these writes are lost, later uses get a backed image. diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index f4cc89f2ba..11e95f98b3 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -167,6 +167,12 @@ class TextureCache : public VideoCommon::ChannelSetupCaches layers); + struct PendingUnswizzle { ImageId image_id; VideoCommon::ImageInfo info; diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index b1fa8db183..39ae25f43a 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -321,6 +321,11 @@ public: return features.features.sparseBinding && graphics_family_sparse_binding; } + /// 2D images whose memory can be bound in parts, on the graphics queue. + bool IsSparseResidencyImage2DSupported() const { + return IsSparseBindingSupported() && features.features.sparseResidencyImage2D; + } + /// Returns true for tile-based deferred renderers. bool IsTiler() const { switch (GetDriverID()) { diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index 92f537aa3c..a0c96d1b45 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -295,6 +295,35 @@ vk::Image MemoryAllocator::CreateUnbackedImage(const VkImageCreateInfo &ci) cons device.GetDispatchLoader()); } +VmaAllocation MemoryAllocator::AllocateSparseMemory(const VkMemoryRequirements &requirements, + VkDeviceMemory &memory, + VkDeviceSize &offset) const { + const VmaAllocationCreateInfo alloc_ci{ + .flags = 0, + .usage = VMA_MEMORY_USAGE_UNKNOWN, + .requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, + .preferredFlags = 0, + .memoryTypeBits = requirements.memoryTypeBits, + .pool = VK_NULL_HANDLE, + .pUserData = nullptr, + .priority = 0.f, + }; + VmaAllocation allocation{}; + VmaAllocationInfo info{}; + if (vmaAllocateMemory(allocator, &requirements, &alloc_ci, &allocation, &info) != VK_SUCCESS) { + return VK_NULL_HANDLE; + } + memory = info.deviceMemory; + offset = info.offset; + return allocation; +} + +void MemoryAllocator::FreeSparseMemory(VmaAllocation allocation) const { + if (allocation != VK_NULL_HANDLE) { + vmaFreeMemory(allocator, allocation); + } +} + vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const { // MESA will do memcpy() if not marked as host cached, so just force mark it for most buffers auto const anv_flags = (usage == MemoryUsage::Stream diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index 99649b9aef..f3c6c177a2 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -111,6 +111,12 @@ namespace Vulkan { /// a null image when the device can't make one. vk::Image CreateUnbackedImage(const VkImageCreateInfo &ci) const; + /// Device memory to bind to part of a sparse resource. Null when there is none left. + VmaAllocation AllocateSparseMemory(const VkMemoryRequirements &requirements, + VkDeviceMemory &memory, VkDeviceSize &offset) const; + + void FreeSparseMemory(VmaAllocation allocation) const; + vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const; vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage, diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index bb43fe399f..dbaaf450a0 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -219,6 +219,7 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkGetEventStatus); X(vkGetFenceStatus); X(vkGetImageMemoryRequirements); + X(vkGetImageSparseMemoryRequirements); X(vkGetPipelineCacheData); X(vkGetMemoryFdKHR); #ifdef _WIN32 @@ -866,6 +867,16 @@ VkMemoryRequirements Device::GetImageMemoryRequirements(VkImage image) const noe return requirements; } +std::vector Device::GetImageSparseMemoryRequirements( + VkImage image) const { + u32 count = 0; + dld->vkGetImageSparseMemoryRequirements(handle, image, &count, nullptr); + std::vector requirements(count); + dld->vkGetImageSparseMemoryRequirements(handle, image, &count, requirements.data()); + requirements.resize(count); + return requirements; +} + std::vector Device::GetPipelineExecutablePropertiesKHR( VkPipeline pipeline) const { const VkPipelineInfoKHR info{ diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index 947150648d..3010368973 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -335,6 +335,7 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkGetEventStatus vkGetEventStatus{}; PFN_vkGetFenceStatus vkGetFenceStatus{}; PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements{}; + PFN_vkGetImageSparseMemoryRequirements vkGetImageSparseMemoryRequirements{}; PFN_vkGetPipelineCacheData vkGetPipelineCacheData{}; PFN_vkGetMemoryFdKHR vkGetMemoryFdKHR{}; #ifdef _WIN32 @@ -1111,6 +1112,9 @@ public: VkMemoryRequirements GetImageMemoryRequirements(VkImage image) const noexcept; + std::vector GetImageSparseMemoryRequirements( + VkImage image) const; + std::vector GetPipelineExecutablePropertiesKHR( VkPipeline pipeline) const; -- cgit v1.2.3 From 6f5e137ec4eb83bbff9ea15f1ad1a8407cfc0cba Mon Sep 17 00:00:00 2001 From: kraibse <33869366+kraibse@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:14:52 +0100 Subject: texture_cache: log unbacked layers over the image's memory cap --- src/video_core/nx2.h | 2 +- .../renderer_vulkan/vk_texture_cache.cpp | 22 ++++++++++++++++++++++ src/video_core/texture_cache/texture_cache.h | 1 + 3 files changed, 24 insertions(+), 1 deletion(-) (limited to 'src/video_core/texture_cache/texture_cache.h') diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h index 1a3bae93ec..4b0d3d5f62 100644 --- a/src/video_core/nx2.h +++ b/src/video_core/nx2.h @@ -96,7 +96,7 @@ inline void Nx2BackSparseWrite(u64 gpu_address, u64 size) { // Which blocks of each sparse region have been written (got memory), for the texture cache: an // image over blocks nobody wrote is all zeros and needs no device memory. Its own lock, taken last: -// the texture cache asks with its lock held, and nx2:gpu updates it holding none of the GPU's. +// the texture cache asks with its lock held, and nx2:gpu updates it holding only its region mutex. struct Nx2SparseRegionState { u64 size; u64 block_size; diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index b77f9af5f8..9204ade9eb 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -2620,6 +2620,20 @@ std::vector Image::BindLayers(std::span layers) { const VkDeviceSize cap = SparseLayersCap(device); std::vector image_binds; std::vector opaque_binds; + const bool tail_was_bound = s.tail_memory != VK_NULL_HANDLE; + std::vector fresh; // Layers given memory by this call + // A failed bind leaves the layers as they were, so the next call tries them again. + const auto roll_back = [&] { + for (const u32 layer : fresh) { + s.allocator->FreeSparseMemory(s.layer_memory[layer]); + s.layer_memory[layer] = VK_NULL_HANDLE; + s.bound_bytes -= s.layer_bytes; + } + if (s.tail_memory != VK_NULL_HANDLE && !tail_was_bound) { + s.allocator->FreeSparseMemory(s.tail_memory); + s.tail_memory = VK_NULL_HANDLE; + } + }; for (const u32 layer : layers) { if (s.layer_memory[layer] != VK_NULL_HANDLE) { bound.push_back(layer); @@ -2647,6 +2661,7 @@ std::vector Image::BindLayers(std::span layers) { } s.layer_memory[layer] = allocation; s.bound_bytes += s.layer_bytes; + fresh.push_back(layer); for (u32 level = 0; level < tail_lod; ++level) { const u32 width = (std::max)(s.width >> level, 1U); const u32 height = (std::max)(s.height >> level, 1U); @@ -2693,6 +2708,12 @@ std::vector Image::BindLayers(std::span layers) { .memoryOffset = offset, .flags = 0, }); + } else { + LOG_WARNING(Render_Vulkan, "Sparse layered image {:#x}: out of device memory for the " + "mip tail", + gpu_addr); + roll_back(); + return {}; } } if (image_binds.empty() && opaque_binds.empty()) { @@ -2737,6 +2758,7 @@ std::vector Image::BindLayers(std::span layers) { if (result != VK_SUCCESS) { LOG_ERROR(Render_Vulkan, "Sparse layered image {:#x}: binding failed ({})", gpu_addr, static_cast(result)); + roll_back(); return {}; } fence.Wait(); diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 3de82d2eb4..63abffed0f 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -2999,6 +2999,7 @@ void TextureCache

::PrepareImage(ImageId image_id, bool is_modification, bool if (True(image.flags & ImageFlagBits::SparseLayers)) { std::vector layers(static_cast(image.info.resources.layers)); std::iota(layers.begin(), layers.end(), 0U); + // Layers over the image's memory cap stay unbacked (logged once). image.BindLayers(layers); } } -- cgit v1.2.3