diff options
| author | Takoroka <137927095+Takoroka@users.noreply.github.com> | 2026-11-01 01:21:38 -0400 |
|---|---|---|
| committer | kraibse <33869366+kraibse@users.noreply.github.com> | 2026-11-01 17:29:56 +0100 |
| commit | 130c255ed91e2bf8f726b38101cc5fc2abcf3935 (patch) | |
| tree | c211079013aaba75309221f971eeef82241125ae /src/video_core | |
| parent | 1c794af56407f324eedad99478388ba0c9cf9ad6 (diff) | |
vulkan: share one zeroed block between empty sparse blocks on AMD
AMD's drivers can't sample images with unbound memory, so large sparse
images are backed whole there, Mario Kart World's nine sparse masks
included (about 2.9 GB, re-uploaded whole for every tile the game maps).
Where the driver supports sparseResidencyAliased, SparseTiles images are
now created with VK_IMAGE_CREATE_SPARSE_ALIASED_BIT and every block is
bound to one zeroed 64 KiB block when the image is created (the image is
cleared once); a block gets memory of its own when data arrives, as on
other drivers. Nothing is ever left unbound. If the GPU renders into such
an image, every block gets memory of its own first, cleared, and the data
is loaded again, so the shared block stays zero.
EDEN_NX2_SPARSE_ALIASED=1 uses the shared block wherever it is supported,
=0 never.
Radeon 780M (proprietary driver 32.0.31041), Mario Kart World title
screen, 200 s: no hang or device loss, world lit, shared memory 12.7 GB
at the end growing 1.8 GB/min, against 18.7 GB and 5.1 GB/min with the
masks backed whole. With every sparse image unbound instead (no AMD
fallback at all) the static world renders dark there, so the other large
sparse images stay backed whole on AMD.
Diffstat (limited to 'src/video_core')
| -rw-r--r-- | src/video_core/renderer_vulkan/vk_texture_cache.cpp | 227 | ||||
| -rw-r--r-- | src/video_core/renderer_vulkan/vk_texture_cache.h | 21 | ||||
| -rw-r--r-- | src/video_core/texture_cache/texture_cache.h | 26 | ||||
| -rw-r--r-- | src/video_core/texture_cache/texture_cache_base.h | 3 | ||||
| -rw-r--r-- | src/video_core/vulkan_common/vulkan_device.h | 5 | ||||
| -rw-r--r-- | src/video_core/vulkan_common/vulkan_memory_allocator.cpp | 5 | ||||
| -rw-r--r-- | src/video_core/vulkan_common/vulkan_memory_allocator.h | 3 |
7 files changed, 281 insertions, 9 deletions
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 947bbebe88..ddd55de690 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -201,13 +201,39 @@ constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) { return layer_uploads && size >= (32U << 20) && Tegra::Nx2InSparseRegion(gpu_addr, size); } +// How SparseTiles images leave the blocks without data: unbound (reads give zero), or all bound to +// one zeroed block where the driver can't sample unbound memory (AMD) but lets blocks share +// memory. EDEN_NX2_SPARSE_ALIASED=1 takes the shared block wherever it is supported, =0 never. +enum class SparseTilesMode { + None, + Unbound, + Aliased, +}; + +[[nodiscard]] SparseTilesMode SparseTilesModeOf(const Device& device) { + static const char force_aliased = [] { + const char* value = std::getenv("EDEN_NX2_SPARSE_ALIASED"); + return value == nullptr ? '\0' : value[0]; + }(); + if (!device.IsSparseResidencyImage2DSupported()) { + return SparseTilesMode::None; + } + const bool aliased = device.IsSparseResidencyAliasedSupported() && force_aliased != '0'; + if (aliased && force_aliased == '1') { + return SparseTilesMode::Aliased; + } + if (device.CanUseUnbackedSparseImages()) { + return SparseTilesMode::Unbound; + } + return aliased ? SparseTilesMode::Aliased : SparseTilesMode::None; +} + // A large single 2D image in a Switch 2 sparse region (title 1's world-sized R8/RG16/R16 masks, // 170-341 MB, of which the game maps a few MiB) gets memory by sparse block where the guest memory // under it can hold data: unbacked at first, then Image::BindTiles. Formats of one texel per // block that the host samples as they are. [[nodiscard]] bool UsesSparseTiles(const Device& device, const ImageInfo& info, GPUVAddr gpu_addr) { - if (!device.CanUseUnbackedSparseImages() || !device.IsSparseResidencyImage2DSupported() || - info.type != ImageType::e2D || info.resources.layers != 1 || info.num_samples != 1 || + if (SparseTilesModeOf(device) == SparseTilesMode::None || info.type != ImageType::e2D || info.resources.layers != 1 || info.num_samples != 1 || info.size.depth != 1 || info.block.width != 0 || VideoCore::Surface::DefaultBlockWidth(info.format) != 1 || VideoCore::Surface::DefaultBlockHeight(info.format) != 1 || @@ -370,7 +396,7 @@ constexpr std::array<VkFormat, 2> OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM [[nodiscard]] vk::Image MakeImage(const Device& device, const MemoryAllocator& allocator, const ImageInfo& info, std::span<const VkFormat> view_formats, std::optional<VkFormat> format_override = {}, - bool unbacked = false) { + bool unbacked = false, bool aliased = false) { if (info.type == ImageType::Buffer) { return vk::Image{}; } @@ -399,7 +425,7 @@ constexpr std::array<VkFormat, 2> OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM } } if (unbacked) { - if (vk::Image image = allocator.CreateUnbackedImage(image_ci)) { + if (vk::Image image = allocator.CreateUnbackedImage(image_ci, aliased)) { return image; } } @@ -2111,7 +2137,9 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu AstcHostFormat(runtime_.device, info), UsesSparseLayers(runtime_.device, info, gpu_addr_) || UsesSparseTiles(runtime_.device, info, gpu_addr_) || - IsUnwrittenSparse(runtime_.device, info, gpu_addr_))), + IsUnwrittenSparse(runtime_.device, info, gpu_addr_), + UsesSparseTiles(runtime_.device, info, gpu_addr_) && + SparseTilesModeOf(runtime_.device) == SparseTilesMode::Aliased)), aspect_mask(ImageAspectMask(info.format)) { if (original_image.IsUnbacked()) { flags |= VideoCommon::ImageFlagBits::Unbacked; @@ -2688,8 +2716,193 @@ bool Image::InitSparseTiles() { size_t{Common::DivCeil(width, s.tile_width)} * Common::DivCeil(height, s.tile_height), VK_NULL_HANDLE); } - LOG_INFO(Render_Vulkan, "Sparse tiled image {:#x}: {}x{} {} levels, block {}x{}, mip tail from {}", - gpu_addr, s.width, s.height, s.levels, s.tile_width, s.tile_height, s.tail_level); + if (SparseTilesModeOf(device) == SparseTilesMode::Aliased && !BindZeroBlock()) { + LOG_WARNING(Render_Vulkan, "Sparse tiled image {:#x}: no zeroed block to share", gpu_addr); + s.Release(); + s.allocator = nullptr; + s.bound_bytes = 0; + return false; + } + LOG_INFO(Render_Vulkan, "Sparse tiled image {:#x}: {}x{} {} levels, block {}x{}, mip tail from {}{}", + gpu_addr, s.width, s.height, s.levels, s.tile_width, s.tile_height, s.tail_level, + s.zero_block != VK_NULL_HANDLE ? ", empty blocks share a zeroed one" : ""); + return true; +} + +bool Image::BindZeroBlock() { + SparseLayerMemory& s = sparse_layers; + const Device& device = runtime->device; + const u64 tile = s.requirements.alignment; + VkDeviceMemory memory{}; + VkDeviceSize offset{}; + s.zero_block = s.allocator->AllocateSparseMemory( + VkMemoryRequirements{ + .size = tile, + .alignment = tile, + .memoryTypeBits = s.requirements.memoryTypeBits, + }, + memory, offset); + if (s.zero_block == VK_NULL_HANDLE) { + return false; + } + s.bound_bytes += tile; + std::vector<VkSparseImageMemoryBind> image_binds; + for (u32 level = 0; level < s.tail_level; ++level) { + const u32 width = (std::max)(s.width >> level, 1U); + const u32 height = (std::max)(s.height >> level, 1U); + for (u32 y = 0; y < height; y += s.tile_height) { + for (u32 x = 0; x < width; x += s.tile_width) { + image_binds.push_back(VkSparseImageMemoryBind{ + .subresource{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .mipLevel = level, + .arrayLayer = 0, + }, + .offset{static_cast<s32>(x), static_cast<s32>(y), 0}, + .extent{(std::min)(s.tile_width, width - x), (std::min)(s.tile_height, height - y), + 1}, + .memory = memory, + .memoryOffset = offset, + .flags = 0, + }); + } + } + } + // The mip tail gets memory of its own: it is small, and the clear below zeroes it too. + std::vector<VkSparseMemoryBind> opaque_binds; + if (s.tail_level < s.levels) { + const VkMemoryRequirements requirements{ + .size = Common::AlignUp(s.sparse.imageMipTailSize, tile), + .alignment = tile, + .memoryTypeBits = s.requirements.memoryTypeBits, + }; + VkDeviceMemory tail_memory{}; + VkDeviceSize tail_offset{}; + s.tail_memory = s.allocator->AllocateSparseMemory(requirements, tail_memory, tail_offset); + if (s.tail_memory == VK_NULL_HANDLE) { + return false; + } + s.bound_bytes += requirements.size; + opaque_binds.push_back(VkSparseMemoryBind{ + .resourceOffset = s.sparse.imageMipTailOffset, + .size = s.sparse.imageMipTailSize, + .memory = tail_memory, + .memoryOffset = tail_offset, + .flags = 0, + }); + } + const VkSparseImageMemoryBindInfo image_bind_info{ + .image = *original_image, + .bindCount = static_cast<u32>(image_binds.size()), + .pBinds = image_binds.data(), + }; + const VkSparseImageOpaqueMemoryBindInfo opaque_bind_info{ + .image = *original_image, + .bindCount = static_cast<u32>(opaque_binds.size()), + .pBinds = opaque_binds.data(), + }; + const VkBindSparseInfo bind_info{ + .sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO, + .pNext = nullptr, + .waitSemaphoreCount = 0, + .pWaitSemaphores = nullptr, + .bufferBindCount = 0, + .pBufferBinds = nullptr, + .imageOpaqueBindCount = opaque_binds.empty() ? 0U : 1U, + .pImageOpaqueBinds = opaque_binds.empty() ? nullptr : &opaque_bind_info, + .imageBindCount = image_binds.empty() ? 0U : 1U, + .pImageBinds = image_binds.empty() ? nullptr : &image_bind_info, + .signalSemaphoreCount = 0, + .pSignalSemaphores = nullptr, + }; + vk::Fence fence = device.GetLogical().CreateFence(VkFenceCreateInfo{ + .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + }); + VkResult result; + { + std::scoped_lock lock{scheduler->submit_mutex}; + result = device.GetGraphicsQueue().BindSparse(bind_info, *fence); + } + if (result != VK_SUCCESS) { + LOG_ERROR(Render_Vulkan, "Sparse tiled image {:#x}: binding the zeroed block failed ({})", + gpu_addr, static_cast<int>(result)); + return false; + } + fence.Wait(); + // Every block now reads the shared block (and the tail its own memory): zero them all. + ClearTiles(); + return true; +} + +void Image::ClearTiles() { + scheduler->RequestOutsideRenderPassOperationContext(); + const VkImage vk_image = *original_image; + const bool was_initialized = std::exchange(initialized, true); + scheduler->Record([vk_image, was_initialized](vk::CommandBuffer cmdbuf) { + static constexpr VkImageSubresourceRange range{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }; + const VkImageMemoryBarrier to_transfer{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | + VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .oldLayout = was_initialized ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vk_image, + .subresourceRange = range, + }; + const VkImageMemoryBarrier to_general{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | + VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | + VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | + VK_ACCESS_TRANSFER_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vk_image, + .subresourceRange = range, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, + to_transfer); + cmdbuf.ClearColorImage(vk_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VkClearColorValue{}, + range); + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, + to_general); + }); +} + +bool Image::BackTilesWhole() { + SparseLayerMemory& s = sparse_layers; + if (s.zero_block == VK_NULL_HANDLE) { + return true; + } + std::vector<VideoCommon::LevelRect> rects; + for (u32 level = 0; level < s.levels; ++level) { + rects.push_back({static_cast<s32>(level), 0, 0, (std::max)(s.width >> level, 1U), + (std::max)(s.height >> level, 1U)}); + } + if (!BindTiles(rects)) { + return false; + } + // No block reads the shared one any more. + s.allocator->FreeSparseMemory(s.zero_block); + s.zero_block = VK_NULL_HANDLE; + s.bound_bytes -= s.requirements.alignment; + ClearTiles(); return true; } diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 89eb8424eb..43218d6ad3 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -404,6 +404,15 @@ public: return static_cast<s32>(sparse_layers.tail_level); } + /// SparseTiles images whose blocks without memory of their own share one zeroed block. + bool SparseTilesShareZeroBlock() const noexcept { + return sparse_layers.zero_block != VK_NULL_HANDLE; + } + + /// SparseTiles images sharing a zeroed block, before the GPU renders into them: every block + /// gets memory of its own, all cleared (the caller loads the data again). False on failure. + bool BackTilesWhole(); + /// Device memory BindLayers has bound to the image's layers (an unbacked image has no other). u64 SparseBoundBytes() const noexcept { return sparse_layers.bound_bytes; @@ -452,6 +461,7 @@ private: level_tiles = std::move(other.level_tiles); other.level_tiles.clear(); tail_level = other.tail_level; + zero_block = std::exchange(other.zero_block, VK_NULL_HANDLE); tail_memory = std::exchange(other.tail_memory, VK_NULL_HANDLE); bound_bytes = other.bound_bytes; warned = other.warned; @@ -473,6 +483,8 @@ private: } } allocator->FreeSparseMemory(tail_memory); + allocator->FreeSparseMemory(zero_block); + zero_block = VK_NULL_HANDLE; layer_memory.clear(); level_tiles.clear(); tail_memory = VK_NULL_HANDLE; @@ -491,6 +503,9 @@ private: /// SparseTiles: per level before the mip tail, per sparse block (rows), null until bound std::vector<std::vector<VmaAllocation>> level_tiles; u32 tail_level{}; ///< SparseTiles: first level of the mip tail + /// SparseTiles: the zeroed block every block without memory of its own is bound to, where + /// the driver can't sample unbound memory + VmaAllocation zero_block{}; VmaAllocation tail_memory{}; ///< The mip tail all layers share, if one VkDeviceSize bound_bytes{}; bool warned{}; @@ -503,6 +518,12 @@ private: /// Reads the image's sparse requirements; false when it can't be bound by sparse block. bool InitSparseTiles(); + /// Binds one zeroed block to every block of a SparseTiles image (and memory to its mip tail). + bool BindZeroBlock(); + + /// Records a clear of the whole image to zero. + void ClearTiles(); + vk::Image original_image; vk::Image scaled_image; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 4b93cd135b..5f655e5b60 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -1466,6 +1466,20 @@ bool TextureCache<P>::BindSparseTiles(Image& image, std::span<const LevelRect> r } template <class P> +bool TextureCache<P>::BackSparseTilesWhole(Image& image) { + if constexpr (requires { &Image::BackTilesWhole; }) { + const u64 bound_bytes = ImageMemoryBytes(image); + const bool backed = image.BackTilesWhole(); + if (True(image.flags & ImageFlagBits::Registered)) { + total_used_memory += ImageMemoryBytes(image) - bound_bytes; + } + return backed; + } else { + return false; + } +} + +template <class P> bool TextureCache<P>::UploadWrittenLayers(Image& image) { if constexpr (requires { &Runtime::AccelerateImageUploadLayers; }) { const u32 num_layers = static_cast<u32>(image.info.resources.layers); @@ -3090,6 +3104,18 @@ void TextureCache<P>::PrepareImage(ImageId image_id, bool is_modification, bool } } if (is_modification) { + if constexpr (requires { &Image::BackTilesWhole; }) { + if (True(image.flags & ImageFlagBits::SparseTiles) && image.SparseTilesShareZeroBlock()) { + // The render would write the zeroed block all empty blocks share: each gets memory + // of its own first, and the parts that hold data are loaded again. + LOG_INFO(HW_GPU, "GPU write into sparse tiled image {:#x}; backing it whole", + image.gpu_addr); + if (BackSparseTilesWhole(image)) { + image.dirty_ranges.clear(); + RefreshSparseTiles(image); + } + } + } if constexpr (requires { &Image::BindLayers; }) { if (True(image.flags & ImageFlagBits::SparseLayers)) { std::vector<u32> layers(static_cast<size_t>(image.info.resources.layers)); diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index f27e237cfa..141e5b76d5 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -173,6 +173,9 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI /// Image::BindTiles, counting what it binds in total_used_memory. bool BindSparseTiles(Image& image, std::span<const LevelRect> rects); + /// Image::BackTilesWhole, counting what it binds in total_used_memory. + bool BackSparseTilesWhole(Image& image); + /// Uploads the image's dirty layers. False when it needs a full upload instead. bool UploadWrittenLayers(Image& image); diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index abd31ad904..3d2408fc35 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -326,6 +326,11 @@ public: return IsSparseBindingSupported() && features.features.sparseResidencyImage2D; } + /// Whether a sparse image may bind the same memory to several of its blocks. + bool IsSparseResidencyAliasedSupported() const { + return IsSparseResidencyImage2DSupported() && features.features.sparseResidencyAliased; + } + /// Whether images with no memory bound may be sampled and used as attachments. AMD's drivers /// fault on it (the GPU hangs shortly after the first such draw), so large images there are /// backed normally. diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index a0c96d1b45..1798611e26 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -280,12 +280,15 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicat device.GetDispatchLoader()); } -vk::Image MemoryAllocator::CreateUnbackedImage(const VkImageCreateInfo &ci) const { +vk::Image MemoryAllocator::CreateUnbackedImage(const VkImageCreateInfo &ci, bool aliased) const { if (ci.imageType == VK_IMAGE_TYPE_1D) { return vk::Image{}; } VkImageCreateInfo sparse_ci = ci; sparse_ci.flags |= VK_IMAGE_CREATE_SPARSE_BINDING_BIT | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT; + if (aliased) { + sparse_ci.flags |= VK_IMAGE_CREATE_SPARSE_ALIASED_BIT; + } VkImage sparse{}; if (device.GetDispatchLoader().vkCreateImage(*device.GetLogical(), &sparse_ci, nullptr, &sparse) != VK_SUCCESS) { diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index f3c6c177a2..e1c267ab79 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -109,7 +109,8 @@ namespace Vulkan { /// A sparse resident image with nothing bound (reads return zero, writes are dropped), or /// a null image when the device can't make one. - vk::Image CreateUnbackedImage(const VkImageCreateInfo &ci) const; + /// aliased: its sparse blocks may share memory (VK_IMAGE_CREATE_SPARSE_ALIASED_BIT). + vk::Image CreateUnbackedImage(const VkImageCreateInfo &ci, bool aliased = false) const; /// Device memory to bind to part of a sparse resource. Null when there is none left. VmaAllocation AllocateSparseMemory(const VkMemoryRequirements &requirements, |
