aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/video_core/renderer_vulkan/vk_texture_cache.cpp227
-rw-r--r--src/video_core/renderer_vulkan/vk_texture_cache.h21
-rw-r--r--src/video_core/texture_cache/texture_cache.h26
-rw-r--r--src/video_core/texture_cache/texture_cache_base.h3
-rw-r--r--src/video_core/vulkan_common/vulkan_device.h5
-rw-r--r--src/video_core/vulkan_common/vulkan_memory_allocator.cpp5
-rw-r--r--src/video_core/vulkan_common/vulkan_memory_allocator.h3
7 files changed, 281 insertions, 9 deletions
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
index 947bbebe88..ddd55de690 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
@@ -201,13 +201,39 @@ constexpr VkBorderColor ConvertBorderColor(const std::array<float, 4>& color) {
return layer_uploads && size >= (32U << 20) && Tegra::Nx2InSparseRegion(gpu_addr, size);
}
+// How SparseTiles images leave the blocks without data: unbound (reads give zero), or all bound to
+// one zeroed block where the driver can't sample unbound memory (AMD) but lets blocks share
+// memory. EDEN_NX2_SPARSE_ALIASED=1 takes the shared block wherever it is supported, =0 never.
+enum class SparseTilesMode {
+ None,
+ Unbound,
+ Aliased,
+};
+
+[[nodiscard]] SparseTilesMode SparseTilesModeOf(const Device& device) {
+ static const char force_aliased = [] {
+ const char* value = std::getenv("EDEN_NX2_SPARSE_ALIASED");
+ return value == nullptr ? '\0' : value[0];
+ }();
+ if (!device.IsSparseResidencyImage2DSupported()) {
+ return SparseTilesMode::None;
+ }
+ const bool aliased = device.IsSparseResidencyAliasedSupported() && force_aliased != '0';
+ if (aliased && force_aliased == '1') {
+ return SparseTilesMode::Aliased;
+ }
+ if (device.CanUseUnbackedSparseImages()) {
+ return SparseTilesMode::Unbound;
+ }
+ return aliased ? SparseTilesMode::Aliased : SparseTilesMode::None;
+}
+
// A large single 2D image in a Switch 2 sparse region (title 1's world-sized R8/RG16/R16 masks,
// 170-341 MB, of which the game maps a few MiB) gets memory by sparse block where the guest memory
// under it can hold data: unbacked at first, then Image::BindTiles. Formats of one texel per
// block that the host samples as they are.
[[nodiscard]] bool UsesSparseTiles(const Device& device, const ImageInfo& info, GPUVAddr gpu_addr) {
- if (!device.CanUseUnbackedSparseImages() || !device.IsSparseResidencyImage2DSupported() ||
- info.type != ImageType::e2D || info.resources.layers != 1 || info.num_samples != 1 ||
+ if (SparseTilesModeOf(device) == SparseTilesMode::None || info.type != ImageType::e2D || info.resources.layers != 1 || info.num_samples != 1 ||
info.size.depth != 1 || info.block.width != 0 ||
VideoCore::Surface::DefaultBlockWidth(info.format) != 1 ||
VideoCore::Surface::DefaultBlockHeight(info.format) != 1 ||
@@ -370,7 +396,7 @@ constexpr std::array<VkFormat, 2> OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM
[[nodiscard]] vk::Image MakeImage(const Device& device, const MemoryAllocator& allocator,
const ImageInfo& info, std::span<const VkFormat> view_formats,
std::optional<VkFormat> format_override = {},
- bool unbacked = false) {
+ bool unbacked = false, bool aliased = false) {
if (info.type == ImageType::Buffer) {
return vk::Image{};
}
@@ -399,7 +425,7 @@ constexpr std::array<VkFormat, 2> OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM
}
}
if (unbacked) {
- if (vk::Image image = allocator.CreateUnbackedImage(image_ci)) {
+ if (vk::Image image = allocator.CreateUnbackedImage(image_ci, aliased)) {
return image;
}
}
@@ -2111,7 +2137,9 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu
AstcHostFormat(runtime_.device, info),
UsesSparseLayers(runtime_.device, info, gpu_addr_) ||
UsesSparseTiles(runtime_.device, info, gpu_addr_) ||
- IsUnwrittenSparse(runtime_.device, info, gpu_addr_))),
+ IsUnwrittenSparse(runtime_.device, info, gpu_addr_),
+ UsesSparseTiles(runtime_.device, info, gpu_addr_) &&
+ SparseTilesModeOf(runtime_.device) == SparseTilesMode::Aliased)),
aspect_mask(ImageAspectMask(info.format)) {
if (original_image.IsUnbacked()) {
flags |= VideoCommon::ImageFlagBits::Unbacked;
@@ -2688,8 +2716,193 @@ bool Image::InitSparseTiles() {
size_t{Common::DivCeil(width, s.tile_width)} * Common::DivCeil(height, s.tile_height),
VK_NULL_HANDLE);
}
- LOG_INFO(Render_Vulkan, "Sparse tiled image {:#x}: {}x{} {} levels, block {}x{}, mip tail from {}",
- gpu_addr, s.width, s.height, s.levels, s.tile_width, s.tile_height, s.tail_level);
+ if (SparseTilesModeOf(device) == SparseTilesMode::Aliased && !BindZeroBlock()) {
+ LOG_WARNING(Render_Vulkan, "Sparse tiled image {:#x}: no zeroed block to share", gpu_addr);
+ s.Release();
+ s.allocator = nullptr;
+ s.bound_bytes = 0;
+ return false;
+ }
+ LOG_INFO(Render_Vulkan, "Sparse tiled image {:#x}: {}x{} {} levels, block {}x{}, mip tail from {}{}",
+ gpu_addr, s.width, s.height, s.levels, s.tile_width, s.tile_height, s.tail_level,
+ s.zero_block != VK_NULL_HANDLE ? ", empty blocks share a zeroed one" : "");
+ return true;
+}
+
+bool Image::BindZeroBlock() {
+ SparseLayerMemory& s = sparse_layers;
+ const Device& device = runtime->device;
+ const u64 tile = s.requirements.alignment;
+ VkDeviceMemory memory{};
+ VkDeviceSize offset{};
+ s.zero_block = s.allocator->AllocateSparseMemory(
+ VkMemoryRequirements{
+ .size = tile,
+ .alignment = tile,
+ .memoryTypeBits = s.requirements.memoryTypeBits,
+ },
+ memory, offset);
+ if (s.zero_block == VK_NULL_HANDLE) {
+ return false;
+ }
+ s.bound_bytes += tile;
+ std::vector<VkSparseImageMemoryBind> image_binds;
+ for (u32 level = 0; level < s.tail_level; ++level) {
+ const u32 width = (std::max)(s.width >> level, 1U);
+ const u32 height = (std::max)(s.height >> level, 1U);
+ for (u32 y = 0; y < height; y += s.tile_height) {
+ for (u32 x = 0; x < width; x += s.tile_width) {
+ image_binds.push_back(VkSparseImageMemoryBind{
+ .subresource{
+ .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+ .mipLevel = level,
+ .arrayLayer = 0,
+ },
+ .offset{static_cast<s32>(x), static_cast<s32>(y), 0},
+ .extent{(std::min)(s.tile_width, width - x), (std::min)(s.tile_height, height - y),
+ 1},
+ .memory = memory,
+ .memoryOffset = offset,
+ .flags = 0,
+ });
+ }
+ }
+ }
+ // The mip tail gets memory of its own: it is small, and the clear below zeroes it too.
+ std::vector<VkSparseMemoryBind> opaque_binds;
+ if (s.tail_level < s.levels) {
+ const VkMemoryRequirements requirements{
+ .size = Common::AlignUp(s.sparse.imageMipTailSize, tile),
+ .alignment = tile,
+ .memoryTypeBits = s.requirements.memoryTypeBits,
+ };
+ VkDeviceMemory tail_memory{};
+ VkDeviceSize tail_offset{};
+ s.tail_memory = s.allocator->AllocateSparseMemory(requirements, tail_memory, tail_offset);
+ if (s.tail_memory == VK_NULL_HANDLE) {
+ return false;
+ }
+ s.bound_bytes += requirements.size;
+ opaque_binds.push_back(VkSparseMemoryBind{
+ .resourceOffset = s.sparse.imageMipTailOffset,
+ .size = s.sparse.imageMipTailSize,
+ .memory = tail_memory,
+ .memoryOffset = tail_offset,
+ .flags = 0,
+ });
+ }
+ const VkSparseImageMemoryBindInfo image_bind_info{
+ .image = *original_image,
+ .bindCount = static_cast<u32>(image_binds.size()),
+ .pBinds = image_binds.data(),
+ };
+ const VkSparseImageOpaqueMemoryBindInfo opaque_bind_info{
+ .image = *original_image,
+ .bindCount = static_cast<u32>(opaque_binds.size()),
+ .pBinds = opaque_binds.data(),
+ };
+ const VkBindSparseInfo bind_info{
+ .sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO,
+ .pNext = nullptr,
+ .waitSemaphoreCount = 0,
+ .pWaitSemaphores = nullptr,
+ .bufferBindCount = 0,
+ .pBufferBinds = nullptr,
+ .imageOpaqueBindCount = opaque_binds.empty() ? 0U : 1U,
+ .pImageOpaqueBinds = opaque_binds.empty() ? nullptr : &opaque_bind_info,
+ .imageBindCount = image_binds.empty() ? 0U : 1U,
+ .pImageBinds = image_binds.empty() ? nullptr : &image_bind_info,
+ .signalSemaphoreCount = 0,
+ .pSignalSemaphores = nullptr,
+ };
+ vk::Fence fence = device.GetLogical().CreateFence(VkFenceCreateInfo{
+ .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO,
+ .pNext = nullptr,
+ .flags = 0,
+ });
+ VkResult result;
+ {
+ std::scoped_lock lock{scheduler->submit_mutex};
+ result = device.GetGraphicsQueue().BindSparse(bind_info, *fence);
+ }
+ if (result != VK_SUCCESS) {
+ LOG_ERROR(Render_Vulkan, "Sparse tiled image {:#x}: binding the zeroed block failed ({})",
+ gpu_addr, static_cast<int>(result));
+ return false;
+ }
+ fence.Wait();
+ // Every block now reads the shared block (and the tail its own memory): zero them all.
+ ClearTiles();
+ return true;
+}
+
+void Image::ClearTiles() {
+ scheduler->RequestOutsideRenderPassOperationContext();
+ const VkImage vk_image = *original_image;
+ const bool was_initialized = std::exchange(initialized, true);
+ scheduler->Record([vk_image, was_initialized](vk::CommandBuffer cmdbuf) {
+ static constexpr VkImageSubresourceRange range{
+ .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+ .baseMipLevel = 0,
+ .levelCount = VK_REMAINING_MIP_LEVELS,
+ .baseArrayLayer = 0,
+ .layerCount = VK_REMAINING_ARRAY_LAYERS,
+ };
+ const VkImageMemoryBarrier to_transfer{
+ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+ .pNext = nullptr,
+ .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT |
+ VK_ACCESS_TRANSFER_WRITE_BIT,
+ .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
+ .oldLayout = was_initialized ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED,
+ .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
+ .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+ .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+ .image = vk_image,
+ .subresourceRange = range,
+ };
+ const VkImageMemoryBarrier to_general{
+ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+ .pNext = nullptr,
+ .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
+ .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT |
+ VK_ACCESS_COLOR_ATTACHMENT_READ_BIT |
+ VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT |
+ VK_ACCESS_TRANSFER_WRITE_BIT,
+ .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
+ .newLayout = VK_IMAGE_LAYOUT_GENERAL,
+ .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+ .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+ .image = vk_image,
+ .subresourceRange = range,
+ };
+ cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0,
+ to_transfer);
+ cmdbuf.ClearColorImage(vk_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VkClearColorValue{},
+ range);
+ cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0,
+ to_general);
+ });
+}
+
+bool Image::BackTilesWhole() {
+ SparseLayerMemory& s = sparse_layers;
+ if (s.zero_block == VK_NULL_HANDLE) {
+ return true;
+ }
+ std::vector<VideoCommon::LevelRect> rects;
+ for (u32 level = 0; level < s.levels; ++level) {
+ rects.push_back({static_cast<s32>(level), 0, 0, (std::max)(s.width >> level, 1U),
+ (std::max)(s.height >> level, 1U)});
+ }
+ if (!BindTiles(rects)) {
+ return false;
+ }
+ // No block reads the shared one any more.
+ s.allocator->FreeSparseMemory(s.zero_block);
+ s.zero_block = VK_NULL_HANDLE;
+ s.bound_bytes -= s.requirements.alignment;
+ ClearTiles();
return true;
}
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h
index 89eb8424eb..43218d6ad3 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.h
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.h
@@ -404,6 +404,15 @@ public:
return static_cast<s32>(sparse_layers.tail_level);
}
+ /// SparseTiles images whose blocks without memory of their own share one zeroed block.
+ bool SparseTilesShareZeroBlock() const noexcept {
+ return sparse_layers.zero_block != VK_NULL_HANDLE;
+ }
+
+ /// SparseTiles images sharing a zeroed block, before the GPU renders into them: every block
+ /// gets memory of its own, all cleared (the caller loads the data again). False on failure.
+ bool BackTilesWhole();
+
/// Device memory BindLayers has bound to the image's layers (an unbacked image has no other).
u64 SparseBoundBytes() const noexcept {
return sparse_layers.bound_bytes;
@@ -452,6 +461,7 @@ private:
level_tiles = std::move(other.level_tiles);
other.level_tiles.clear();
tail_level = other.tail_level;
+ zero_block = std::exchange(other.zero_block, VK_NULL_HANDLE);
tail_memory = std::exchange(other.tail_memory, VK_NULL_HANDLE);
bound_bytes = other.bound_bytes;
warned = other.warned;
@@ -473,6 +483,8 @@ private:
}
}
allocator->FreeSparseMemory(tail_memory);
+ allocator->FreeSparseMemory(zero_block);
+ zero_block = VK_NULL_HANDLE;
layer_memory.clear();
level_tiles.clear();
tail_memory = VK_NULL_HANDLE;
@@ -491,6 +503,9 @@ private:
/// SparseTiles: per level before the mip tail, per sparse block (rows), null until bound
std::vector<std::vector<VmaAllocation>> level_tiles;
u32 tail_level{}; ///< SparseTiles: first level of the mip tail
+ /// SparseTiles: the zeroed block every block without memory of its own is bound to, where
+ /// the driver can't sample unbound memory
+ VmaAllocation zero_block{};
VmaAllocation tail_memory{}; ///< The mip tail all layers share, if one
VkDeviceSize bound_bytes{};
bool warned{};
@@ -503,6 +518,12 @@ private:
/// Reads the image's sparse requirements; false when it can't be bound by sparse block.
bool InitSparseTiles();
+ /// Binds one zeroed block to every block of a SparseTiles image (and memory to its mip tail).
+ bool BindZeroBlock();
+
+ /// Records a clear of the whole image to zero.
+ void ClearTiles();
+
vk::Image original_image;
vk::Image scaled_image;
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 4b93cd135b..5f655e5b60 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -1466,6 +1466,20 @@ bool TextureCache<P>::BindSparseTiles(Image& image, std::span<const LevelRect> r
}
template <class P>
+bool TextureCache<P>::BackSparseTilesWhole(Image& image) {
+ if constexpr (requires { &Image::BackTilesWhole; }) {
+ const u64 bound_bytes = ImageMemoryBytes(image);
+ const bool backed = image.BackTilesWhole();
+ if (True(image.flags & ImageFlagBits::Registered)) {
+ total_used_memory += ImageMemoryBytes(image) - bound_bytes;
+ }
+ return backed;
+ } else {
+ return false;
+ }
+}
+
+template <class P>
bool TextureCache<P>::UploadWrittenLayers(Image& image) {
if constexpr (requires { &Runtime::AccelerateImageUploadLayers; }) {
const u32 num_layers = static_cast<u32>(image.info.resources.layers);
@@ -3090,6 +3104,18 @@ void TextureCache<P>::PrepareImage(ImageId image_id, bool is_modification, bool
}
}
if (is_modification) {
+ if constexpr (requires { &Image::BackTilesWhole; }) {
+ if (True(image.flags & ImageFlagBits::SparseTiles) && image.SparseTilesShareZeroBlock()) {
+ // The render would write the zeroed block all empty blocks share: each gets memory
+ // of its own first, and the parts that hold data are loaded again.
+ LOG_INFO(HW_GPU, "GPU write into sparse tiled image {:#x}; backing it whole",
+ image.gpu_addr);
+ if (BackSparseTilesWhole(image)) {
+ image.dirty_ranges.clear();
+ RefreshSparseTiles(image);
+ }
+ }
+ }
if constexpr (requires { &Image::BindLayers; }) {
if (True(image.flags & ImageFlagBits::SparseLayers)) {
std::vector<u32> layers(static_cast<size_t>(image.info.resources.layers));
diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h
index f27e237cfa..141e5b76d5 100644
--- a/src/video_core/texture_cache/texture_cache_base.h
+++ b/src/video_core/texture_cache/texture_cache_base.h
@@ -173,6 +173,9 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI
/// Image::BindTiles, counting what it binds in total_used_memory.
bool BindSparseTiles(Image& image, std::span<const LevelRect> rects);
+ /// Image::BackTilesWhole, counting what it binds in total_used_memory.
+ bool BackSparseTilesWhole(Image& image);
+
/// Uploads the image's dirty layers. False when it needs a full upload instead.
bool UploadWrittenLayers(Image& image);
diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h
index abd31ad904..3d2408fc35 100644
--- a/src/video_core/vulkan_common/vulkan_device.h
+++ b/src/video_core/vulkan_common/vulkan_device.h
@@ -326,6 +326,11 @@ public:
return IsSparseBindingSupported() && features.features.sparseResidencyImage2D;
}
+ /// Whether a sparse image may bind the same memory to several of its blocks.
+ bool IsSparseResidencyAliasedSupported() const {
+ return IsSparseResidencyImage2DSupported() && features.features.sparseResidencyAliased;
+ }
+
/// Whether images with no memory bound may be sampled and used as attachments. AMD's drivers
/// fault on it (the GPU hangs shortly after the first such draw), so large images there are
/// backed normally.
diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
index a0c96d1b45..1798611e26 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp
@@ -280,12 +280,15 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci, bool dedicat
device.GetDispatchLoader());
}
-vk::Image MemoryAllocator::CreateUnbackedImage(const VkImageCreateInfo &ci) const {
+vk::Image MemoryAllocator::CreateUnbackedImage(const VkImageCreateInfo &ci, bool aliased) const {
if (ci.imageType == VK_IMAGE_TYPE_1D) {
return vk::Image{};
}
VkImageCreateInfo sparse_ci = ci;
sparse_ci.flags |= VK_IMAGE_CREATE_SPARSE_BINDING_BIT | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT;
+ if (aliased) {
+ sparse_ci.flags |= VK_IMAGE_CREATE_SPARSE_ALIASED_BIT;
+ }
VkImage sparse{};
if (device.GetDispatchLoader().vkCreateImage(*device.GetLogical(), &sparse_ci, nullptr,
&sparse) != VK_SUCCESS) {
diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h
index f3c6c177a2..e1c267ab79 100644
--- a/src/video_core/vulkan_common/vulkan_memory_allocator.h
+++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h
@@ -109,7 +109,8 @@ namespace Vulkan {
/// A sparse resident image with nothing bound (reads return zero, writes are dropped), or
/// a null image when the device can't make one.
- vk::Image CreateUnbackedImage(const VkImageCreateInfo &ci) const;
+ /// aliased: its sparse blocks may share memory (VK_IMAGE_CREATE_SPARSE_ALIASED_BIT).
+ vk::Image CreateUnbackedImage(const VkImageCreateInfo &ci, bool aliased = false) const;
/// Device memory to bind to part of a sparse resource. Null when there is none left.
VmaAllocation AllocateSparseMemory(const VkMemoryRequirements &requirements,
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-04 03:48:34 +0000