aboutsummaryrefslogtreecommitdiff
path: root/src/video_core/renderer_vulkan
diff options
context:
space:
mode:
authorkraibse <33869366+kraibse@users.noreply.github.com>2026-11-02 16:01:57 +0100
committerkraibse <33869366+kraibse@users.noreply.github.com>2026-11-02 16:01:57 +0100
commitb374709aa202d586dd09f81f3389da8d73fea0d7 (patch)
tree5c4033d827c517aca9cc73dc497fbb3e81291bf1 /src/video_core/renderer_vulkan
parent021c9724371e45fa694c9ee3106b9011364d2853 (diff)
parentce1b1de6f1beed36958368f89cda466937ff9716 (diff)
Merge remote-tracking branch 'origin/nx2' into codex/upstream-loading-hotkeys
Diffstat (limited to 'src/video_core/renderer_vulkan')
-rw-r--r--src/video_core/renderer_vulkan/blit_image.cpp147
-rw-r--r--src/video_core/renderer_vulkan/blit_image.h15
-rw-r--r--src/video_core/renderer_vulkan/fixed_pipeline_state.cpp13
-rw-r--r--src/video_core/renderer_vulkan/maxwell_to_vk.cpp4
-rw-r--r--src/video_core/renderer_vulkan/present/layer.cpp2
-rw-r--r--src/video_core/renderer_vulkan/vertex_repack.h99
-rw-r--r--src/video_core/renderer_vulkan/vk_buffer_cache.cpp51
-rw-r--r--src/video_core/renderer_vulkan/vk_buffer_cache.h8
-rw-r--r--src/video_core/renderer_vulkan/vk_compute_pass.cpp64
-rw-r--r--src/video_core/renderer_vulkan/vk_compute_pass.h14
-rw-r--r--src/video_core/renderer_vulkan/vk_compute_pipeline.cpp18
-rw-r--r--src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp57
-rw-r--r--src/video_core/renderer_vulkan/vk_graphics_pipeline.h21
-rw-r--r--src/video_core/renderer_vulkan/vk_pipeline_cache.cpp222
-rw-r--r--src/video_core/renderer_vulkan/vk_pipeline_cache.h21
-rw-r--r--src/video_core/renderer_vulkan/vk_rasterizer.cpp57
-rw-r--r--src/video_core/renderer_vulkan/vk_rasterizer.h6
-rw-r--r--src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp16
-rw-r--r--src/video_core/renderer_vulkan/vk_texture_cache.cpp317
-rw-r--r--src/video_core/renderer_vulkan/vk_texture_cache.h41
-rw-r--r--src/video_core/renderer_vulkan/vk_update_descriptor.cpp8
-rw-r--r--src/video_core/renderer_vulkan/vk_update_descriptor.h3
22 files changed, 1154 insertions, 50 deletions
diff --git a/src/video_core/renderer_vulkan/blit_image.cpp b/src/video_core/renderer_vulkan/blit_image.cpp
index ddd52e9426..edd151b563 100644
--- a/src/video_core/renderer_vulkan/blit_image.cpp
+++ b/src/video_core/renderer_vulkan/blit_image.cpp
@@ -33,6 +33,7 @@
#include "video_core/host_shaders/convert_non_msaa_to_msaa_depth_stencil_frag_spv.h"
#include "video_core/host_shaders/convert_s8d24_to_abgr8_frag_spv.h"
#include "video_core/host_shaders/full_screen_triangle_vert_spv.h"
+#include "video_core/host_shaders/temporal_upscale_frag_spv.h"
#include "video_core/host_shaders/vulkan_blit_depth_stencil_frag_spv.h"
#include "video_core/host_shaders/vulkan_color_clear_frag_spv.h"
#include "video_core/host_shaders/vulkan_color_clear_vert_spv.h"
@@ -91,6 +92,15 @@ struct PushConstants {
std::array<float, 2> tex_offset;
};
+// The vertex stage's PushConstants first, then those of temporal_upscale.frag.
+struct TemporalUpscalePushConstants {
+ PushConstants vertex;
+ std::array<float, 2> jitter;
+ std::array<float, 2> motion_scale;
+ float current_weight;
+ float padding;
+};
+
struct MSAACopyPushConstants {
std::array<s32, 2> dst_offset;
std::array<s32, 2> src_offset;
@@ -109,6 +119,11 @@ constexpr std::array TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS{
TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<0>,
TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<1>,
};
+constexpr std::array THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS{
+ TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<0>,
+ TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<1>,
+ TEXTURE_DESCRIPTOR_SET_LAYOUT_BINDING<2>,
+};
constexpr VkDescriptorSetLayoutCreateInfo ONE_TEXTURE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO{
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
.pNext = nullptr,
@@ -133,6 +148,13 @@ constexpr VkDescriptorSetLayoutCreateInfo TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_CRE
.bindingCount = static_cast<u32>(TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.size()),
.pBindings = TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.data(),
};
+constexpr VkDescriptorSetLayoutCreateInfo THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO{
+ .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
+ .pNext = nullptr,
+ .flags = 0,
+ .bindingCount = static_cast<u32>(THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.size()),
+ .pBindings = THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_BINDINGS.data(),
+};
template <VkShaderStageFlags stageFlags, size_t size>
inline constexpr VkPushConstantRange PUSH_CONSTANT_RANGE{
.stageFlags = stageFlags,
@@ -357,6 +379,32 @@ void UpdateOneTextureDescriptorSet(const Device& device, VkDescriptorSet descrip
device.GetLogical().UpdateDescriptorSets(write_descriptor_set, nullptr);
}
+void UpdateThreeTexturesDescriptorSet(const Device& device, VkDescriptorSet descriptor_set,
+ VkSampler sampler, std::array<VkImageView, 3> image_views) {
+ std::array<VkDescriptorImageInfo, 3> image_infos{};
+ std::array<VkWriteDescriptorSet, 3> write_descriptor_sets{};
+ for (u32 binding = 0; binding < 3; ++binding) {
+ image_infos[binding] = VkDescriptorImageInfo{
+ .sampler = sampler,
+ .imageView = image_views[binding],
+ .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
+ };
+ write_descriptor_sets[binding] = VkWriteDescriptorSet{
+ .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
+ .pNext = nullptr,
+ .dstSet = descriptor_set,
+ .dstBinding = binding,
+ .dstArrayElement = 0,
+ .descriptorCount = 1,
+ .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
+ .pImageInfo = &image_infos[binding],
+ .pBufferInfo = nullptr,
+ .pTexelBufferView = nullptr,
+ };
+ }
+ device.GetLogical().UpdateDescriptorSets(write_descriptor_sets, nullptr);
+}
+
void UpdateTwoTexturesDescriptorSet(const Device& device, VkDescriptorSet descriptor_set,
VkSampler sampler, VkImageView image_view_0,
VkImageView image_view_1) {
@@ -590,10 +638,14 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_,
ONE_TEXTURE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)),
two_textures_set_layout(device.GetLogical().CreateDescriptorSetLayout(
TWO_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)),
+ three_textures_set_layout(device.GetLogical().CreateDescriptorSetLayout(
+ THREE_TEXTURES_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)),
one_texture_descriptor_allocator{
descriptor_pool.Allocator(device_, scheduler_, *one_texture_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<1>)},
two_textures_descriptor_allocator{
descriptor_pool.Allocator(device_, scheduler_, *two_textures_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<2>)},
+ three_textures_descriptor_allocator{
+ descriptor_pool.Allocator(device_, scheduler_, *three_textures_set_layout, TEXTURE_DESCRIPTOR_BANK_INFO<3>)},
one_texture_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo(
one_texture_set_layout.address(),
PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_VERTEX_BIT, sizeof(PushConstants)>))),
@@ -601,6 +653,11 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_,
device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo(
two_textures_set_layout.address(),
PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_VERTEX_BIT, sizeof(PushConstants)>))),
+ temporal_upscale_pipeline_layout(
+ device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo(
+ three_textures_set_layout.address(),
+ PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT,
+ sizeof(TemporalUpscalePushConstants)>))),
clear_color_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo(
nullptr, PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_FRAGMENT_BIT, sizeof(float) * 4>))),
msaa_copy_pipeline_layout(device.GetLogical().CreatePipelineLayout(PipelineLayoutCreateInfo(
@@ -612,6 +669,7 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_,
PUSH_CONSTANT_RANGE<VK_SHADER_STAGE_FRAGMENT_BIT, sizeof(MSAACopyPushConstants)>))),
full_screen_vert(BuildShader(device, FULL_SCREEN_TRIANGLE_VERT_SPV)),
blit_color_to_color_frag(BuildShader(device, BLIT_COLOR_FLOAT_FRAG_SPV)),
+ temporal_upscale_frag(BuildShader(device, TEMPORAL_UPSCALE_FRAG_SPV)),
blit_color_msaa_frag(BuildShader(device, BLIT_COLOR_MSAA_FRAG_SPV)),
blit_depth_stencil_frag(device.IsExtShaderStencilExportSupported()
? BuildShader(device, VULKAN_BLIT_DEPTH_STENCIL_FRAG_SPV)
@@ -702,6 +760,42 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, VkImageView
});
}
+void BlitImageHelper::TemporalUpscale(const Framebuffer* dst_framebuffer,
+ const ImageView& input_color, const ImageView& motion_vectors,
+ VkImageView history, const Region2D& dst_region,
+ std::array<f32, 2> jitter, std::array<f32, 2> motion_scale,
+ f32 current_weight) {
+ const VkPipelineLayout layout = *temporal_upscale_pipeline_layout;
+ const VkPipeline pipeline = FindOrEmplaceTemporalUpscalePipeline(dst_framebuffer->RenderPass());
+ const VkSampler sampler = *linear_sampler;
+ const std::array<VkImageView, 3> views{input_color.Handle(Shader::TextureType::Color2D),
+ motion_vectors.Handle(Shader::TextureType::Color2D),
+ history};
+ const TemporalUpscalePushConstants push_constants{
+ .vertex = {.tex_scale = {1.0f, 1.0f}, .tex_offset = {0.0f, 0.0f}},
+ .jitter = jitter,
+ .motion_scale = motion_scale,
+ .current_weight = current_weight,
+ .padding = 0.0f,
+ };
+ RecordShaderReadBarrier(scheduler, input_color);
+ RecordShaderReadBarrier(scheduler, motion_vectors);
+ scheduler.RequestRenderpass(dst_framebuffer);
+ scheduler.Record([this, dst_region, pipeline, layout, sampler, views,
+ push_constants](vk::CommandBuffer cmdbuf) {
+ const VkDescriptorSet descriptor_set = three_textures_descriptor_allocator.Commit();
+ UpdateThreeTexturesDescriptorSet(device, descriptor_set, sampler, views);
+ cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
+ cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, descriptor_set,
+ nullptr);
+ BindBlitState(cmdbuf, dst_region);
+ cmdbuf.PushConstants(layout, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT,
+ push_constants);
+ cmdbuf.Draw(3, 1, 0, 0);
+ });
+ scheduler.InvalidateState();
+}
+
void BlitImageHelper::BlitImpl(const Framebuffer* dst_framebuffer,
const ImageView& src_image_view, const Region2D& dst_region,
const Region2D& src_region, VkPipeline pipeline, VkSampler sampler,
@@ -1285,6 +1379,59 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe
return *blit_color_pipelines.back();
}
+VkPipeline BlitImageHelper::FindOrEmplaceTemporalUpscalePipeline(VkRenderPass renderpass) {
+ const auto it = std::ranges::find(temporal_upscale_keys, renderpass);
+ if (it != temporal_upscale_keys.end()) {
+ return *temporal_upscale_pipelines[std::distance(temporal_upscale_keys.begin(), it)];
+ }
+ temporal_upscale_keys.push_back(renderpass);
+ const std::array stages = MakeStages(*full_screen_vert, *temporal_upscale_frag);
+ const VkPipelineColorBlendAttachmentState blend_attachment{
+ .blendEnable = VK_FALSE,
+ .srcColorBlendFactor = VK_BLEND_FACTOR_ZERO,
+ .dstColorBlendFactor = VK_BLEND_FACTOR_ZERO,
+ .colorBlendOp = VK_BLEND_OP_ADD,
+ .srcAlphaBlendFactor = VK_BLEND_FACTOR_ZERO,
+ .dstAlphaBlendFactor = VK_BLEND_FACTOR_ZERO,
+ .alphaBlendOp = VK_BLEND_OP_ADD,
+ .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT |
+ VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT,
+ };
+ const VkPipelineColorBlendStateCreateInfo color_blend_create_info{
+ .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
+ .pNext = nullptr,
+ .flags = 0,
+ .logicOpEnable = VK_FALSE,
+ .logicOp = VK_LOGIC_OP_CLEAR,
+ .attachmentCount = 1,
+ .pAttachments = &blend_attachment,
+ .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f},
+ };
+ const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device);
+ temporal_upscale_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({
+ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
+ .pNext = nullptr,
+ .flags = 0,
+ .stageCount = static_cast<u32>(stages.size()),
+ .pStages = stages.data(),
+ .pVertexInputState = &PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
+ .pInputAssemblyState = &input_assembly_ci,
+ .pTessellationState = nullptr,
+ .pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO,
+ .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
+ .pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
+ .pDepthStencilState = nullptr,
+ .pColorBlendState = &color_blend_create_info,
+ .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO,
+ .layout = *temporal_upscale_pipeline_layout,
+ .renderPass = renderpass,
+ .subpass = 0,
+ .basePipelineHandle = VK_NULL_HANDLE,
+ .basePipelineIndex = 0,
+ }, device.StaticPipelineCache()));
+ return *temporal_upscale_pipelines.back();
+}
+
VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key) {
const auto it = std::ranges::find(blit_depth_stencil_keys, key);
if (it != blit_depth_stencil_keys.end()) {
diff --git a/src/video_core/renderer_vulkan/blit_image.h b/src/video_core/renderer_vulkan/blit_image.h
index 62e6743a23..303beeea5c 100644
--- a/src/video_core/renderer_vulkan/blit_image.h
+++ b/src/video_core/renderer_vulkan/blit_image.h
@@ -82,6 +82,13 @@ public:
VkImage src_image, VkSampler src_sampler, const Region2D& dst_region,
const Region2D& src_region, const Extent3D& src_size);
+ /// Temporal upscale (temporal_upscale.frag) into dst_framebuffer: the jittered input colour,
+ /// accumulated with the history image reprojected along the motion vectors.
+ void TemporalUpscale(const Framebuffer* dst_framebuffer, const ImageView& input_color,
+ const ImageView& motion_vectors, VkImageView history,
+ const Region2D& dst_region, std::array<f32, 2> jitter,
+ std::array<f32, 2> motion_scale, f32 current_weight);
+
void BlitColorMSAA(const Framebuffer* dst_framebuffer, const ImageView& src_image_view,
const Region2D& dst_region, const Region2D& src_region);
@@ -170,6 +177,8 @@ private:
[[nodiscard]] VkPipeline FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key);
+ [[nodiscard]] VkPipeline FindOrEmplaceTemporalUpscalePipeline(VkRenderPass renderpass);
+
[[nodiscard]] VkPipeline FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key);
[[nodiscard]] VkPipeline FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key);
@@ -207,15 +216,19 @@ private:
vk::DescriptorSetLayout one_texture_set_layout;
vk::DescriptorSetLayout two_textures_set_layout;
+ vk::DescriptorSetLayout three_textures_set_layout;
DescriptorAllocator one_texture_descriptor_allocator;
DescriptorAllocator two_textures_descriptor_allocator;
+ DescriptorAllocator three_textures_descriptor_allocator;
vk::PipelineLayout one_texture_pipeline_layout;
vk::PipelineLayout two_textures_pipeline_layout;
+ vk::PipelineLayout temporal_upscale_pipeline_layout;
vk::PipelineLayout clear_color_pipeline_layout;
vk::PipelineLayout msaa_copy_pipeline_layout;
vk::PipelineLayout msaa_copy_depth_stencil_pipeline_layout;
vk::ShaderModule full_screen_vert;
vk::ShaderModule blit_color_to_color_frag;
+ vk::ShaderModule temporal_upscale_frag;
vk::ShaderModule blit_color_msaa_frag;
vk::ShaderModule blit_depth_stencil_frag;
vk::ShaderModule blit_depth_frag;
@@ -246,6 +259,8 @@ private:
std::vector<BlitImagePipelineKey> blit_color_keys;
std::vector<vk::Pipeline> blit_color_pipelines;
+ std::vector<VkRenderPass> temporal_upscale_keys;
+ std::vector<vk::Pipeline> temporal_upscale_pipelines;
std::vector<BlitImagePipelineKey> blit_depth_stencil_keys;
std::vector<vk::Pipeline> blit_depth_stencil_pipelines;
std::vector<BlitImagePipelineKey> clear_color_keys;
diff --git a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp
index 9dac98c830..d1f78c0034 100644
--- a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp
+++ b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp
@@ -14,6 +14,7 @@
#include "common/settings.h"
#include "video_core/engines/maxwell_3d.h"
#include "video_core/renderer_vulkan/fixed_pipeline_state.h"
+#include "video_core/renderer_vulkan/vertex_repack.h"
#include "video_core/renderer_vulkan/vk_state_tracker.h"
namespace Vulkan {
@@ -268,6 +269,18 @@ void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFe
if (xfb_enabled) {
RefreshXfbState(xfb_state, regs);
}
+ if (!dynamic_vertex_input && ExperimentalStaticVertexRepackEnabled()) {
+ const u32 repack_mask = VertexRepackMask(regs);
+ // Refresh from guest values even when only the stride changed: offsets must not accumulate.
+ for (size_t index = 0; index < attributes.size(); ++index) {
+ const auto& input = regs.vertex_attrib_format[index];
+ attributes[index].offset.Assign(RepackedVertexOffset(repack_mask, input.buffer, input.offset));
+ }
+ for (u32 binding = 0; binding < vertex_strides.size(); ++binding) {
+ vertex_strides[binding] = static_cast<u16>(RepackedVertexStride(
+ repack_mask, binding, regs.vertex_streams[binding].stride));
+ }
+ }
}
void FixedPipelineState::BlendingAttachment::Refresh(const Maxwell& regs, size_t index) {
diff --git a/src/video_core/renderer_vulkan/maxwell_to_vk.cpp b/src/video_core/renderer_vulkan/maxwell_to_vk.cpp
index c8e43d0650..e03e5786fe 100644
--- a/src/video_core/renderer_vulkan/maxwell_to_vk.cpp
+++ b/src/video_core/renderer_vulkan/maxwell_to_vk.cpp
@@ -333,6 +333,10 @@ VkShaderStageFlagBits ShaderStage(Shader::Stage stage) {
return VK_SHADER_STAGE_FRAGMENT_BIT;
case Shader::Stage::Compute:
return VK_SHADER_STAGE_COMPUTE_BIT;
+ case Shader::Stage::Task:
+ return VK_SHADER_STAGE_TASK_BIT_EXT;
+ case Shader::Stage::Mesh:
+ return VK_SHADER_STAGE_MESH_BIT_EXT;
}
UNIMPLEMENTED_MSG("Unimplemented shader stage={}", stage);
return {};
diff --git a/src/video_core/renderer_vulkan/present/layer.cpp b/src/video_core/renderer_vulkan/present/layer.cpp
index fa24c02f6d..5652090bd7 100644
--- a/src/video_core/renderer_vulkan/present/layer.cpp
+++ b/src/video_core/renderer_vulkan/present/layer.cpp
@@ -52,6 +52,8 @@ VkFormat GetFormat(const Tegra::FramebufferConfig& framebuffer) {
return VK_FORMAT_R5G6B5_UNORM_PACK16;
case Service::android::PixelFormat::Bgra8888:
return VK_FORMAT_B8G8R8A8_UNORM;
+ case Service::android::PixelFormat::Rgba1010102:
+ return VK_FORMAT_A2B10G10R10_UNORM_PACK32;
default:
UNIMPLEMENTED_MSG("Unknown framebuffer pixel format: {}",
static_cast<u32>(framebuffer.pixel_format));
diff --git a/src/video_core/renderer_vulkan/vertex_repack.h b/src/video_core/renderer_vulkan/vertex_repack.h
new file mode 100644
index 0000000000..b0849c25c7
--- /dev/null
+++ b/src/video_core/renderer_vulkan/vertex_repack.h
@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+#include <optional>
+#include "video_core/engines/maxwell_3d.h"
+
+namespace Vulkan {
+
+constexpr bool VertexRepackInputSupported(bool dynamic_input, bool experimental_static) {
+ return dynamic_input || experimental_static;
+}
+
+inline bool ExperimentalStaticVertexRepackEnabled() {
+ static const bool enabled = [] {
+ const char* value = std::getenv("EGLANTINE_EXPERIMENTAL_STATIC_VERTEX_REPACK");
+ return value && std::strcmp(value, "1") == 0;
+ }();
+ return enabled;
+}
+
+struct VertexRepackParameters {
+ u32 vertices;
+ u32 output_size;
+ u32 source_base;
+ u32 source_bias;
+ u32 source_range;
+ u32 dispatch_groups;
+};
+
+inline std::optional<VertexRepackParameters> MakeVertexRepackParameters(
+ u32 offset, u32 size, u64 alignment, u64 max_storage_range, u32 max_dispatch_groups) {
+ if (size == 0 || alignment == 0) {
+ return std::nullopt;
+ }
+ const u64 vertices = (u64{size} + 68) / 69;
+ const u64 output_size = vertices * 72;
+ const u64 base = u64{offset} / alignment * alignment;
+ const u64 bias = offset - base;
+ const u64 range = (bias + size + 3) / 4 * 4;
+ const u64 groups = (vertices * 18 + 127) / 128;
+ if (output_size > std::numeric_limits<u32>::max() ||
+ range > std::numeric_limits<u32>::max() || output_size > max_storage_range ||
+ range > max_storage_range || groups > max_dispatch_groups) {
+ return std::nullopt;
+ }
+ return VertexRepackParameters{static_cast<u32>(vertices), static_cast<u32>(output_size),
+ static_cast<u32>(base), static_cast<u32>(bias),
+ static_cast<u32>(range), static_cast<u32>(groups)};
+}
+
+inline bool VertexRepackEnabled() {
+ static const bool enabled = [] {
+ const char* value = std::getenv("EGLANTINE_TEST_VERTEX_REPACK");
+ return !value || std::strcmp(value, "0") != 0;
+ }();
+ return enabled;
+}
+
+inline u32 VertexRepackMask(const Tegra::Engines::Maxwell3D::Regs& regs) {
+ using Attribute = Tegra::Engines::Maxwell3D::Regs::VertexAttribute;
+ if (!VertexRepackEnabled()) {
+ return 0;
+ }
+ u32 candidates = 0;
+ u32 excluded = 0;
+ for (const auto& attribute : regs.vertex_attrib_format) {
+ const u32 binding = attribute.buffer;
+ if (attribute.constant || regs.vertex_streams[binding].stride != 69) {
+ continue;
+ }
+ if (attribute.size == Attribute::Size::Invalid) {
+ excluded |= 1U << binding;
+ continue;
+ }
+ const u32 offset = attribute.offset;
+ const u32 size = attribute.SizeInBytes();
+ if (offset == 65 && size == 4) {
+ candidates |= 1U << binding;
+ }
+ if (offset + size > 69 || (offset < 65 && offset + size > 65)) {
+ excluded |= 1U << binding;
+ }
+ }
+ return candidates & ~excluded;
+}
+
+inline u32 RepackedVertexOffset(u32 mask, u32 binding, u32 offset) {
+ return offset + ((mask & (1U << binding)) && offset >= 65 ? 3U : 0U);
+}
+
+inline u32 RepackedVertexStride(u32 mask, u32 binding, u32 stride) {
+ return mask & (1U << binding) ? 72U : stride;
+}
+
+} // namespace Vulkan
diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp
index 493de2cc20..fe0ab3a28e 100644
--- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp
@@ -7,11 +7,13 @@
#include <algorithm>
#include <array>
#include <cstring>
+#include <cstdlib>
#include <span>
#include <vector>
#include "video_core/buffer_cache/buffer_cache_base.h"
#include "video_core/renderer_vulkan/vk_buffer_cache.h"
+#include "video_core/renderer_vulkan/vertex_repack.h"
#include "video_core/renderer_vulkan/maxwell_to_vk.h"
#include "video_core/renderer_vulkan/vk_scheduler.h"
@@ -359,6 +361,14 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m
compute_pass_descriptor_queue),
multi_range_buffers(device_) {
const VkDriverIdKHR driver_id = device.GetDriverID();
+ const bool enable_vertex_repack = VertexRepackEnabled();
+ LOG_INFO(Render_Vulkan, "69-byte vertex layout repacking {} ({})",
+ enable_vertex_repack ? "enabled" : "disabled",
+ "portable Vulkan layout fallback");
+ if (enable_vertex_repack) {
+ vertex_repack_pass = std::make_unique<VertexRepackPass>(
+ device, scheduler, descriptor_pool, staging_pool, compute_pass_descriptor_queue);
+ }
limit_dynamic_storage_buffers = driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY ||
driver_id == VK_DRIVER_ID_ARM_PROPRIETARY;
if (limit_dynamic_storage_buffers) {
@@ -374,6 +384,19 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m
scheduler_, staging_pool_);
}
+u32 BufferCacheRuntime::SetVertexRepackLayout(const Tegra::Engines::Maxwell3D::Regs& regs,
+ bool dynamic_input) {
+ const u32 old_mask = vertex_repack_mask;
+ const bool input_supported = VertexRepackInputSupported(
+ dynamic_input, ExperimentalStaticVertexRepackEnabled());
+ vertex_repack_mask = vertex_repack_pass && input_supported ? VertexRepackMask(regs) : 0;
+ if (vertex_repack_mask != old_mask) {
+ LOG_DEBUG(Render_Vulkan, "Vertex repack selection: mask={:#x}, dynamic_input={}",
+ vertex_repack_mask, dynamic_input);
+ }
+ return old_mask | vertex_repack_mask;
+}
+
StagingBufferRef BufferCacheRuntime::UploadStagingBuffer(size_t size) {
return staging_pool.Request(size, MemoryUsage::Upload);
}
@@ -629,6 +652,16 @@ void BufferCacheRuntime::BindVertexBuffer(u32 index, VkBuffer buffer, u32 offset
if (index >= device.GetMaxVertexInputBindings()) {
return;
}
+ if (buffer != VK_NULL_HANDLE && size != 0 && IsVertexRepacked(index)) {
+ const auto repacked = vertex_repack_pass->Assemble(buffer, offset, size);
+ if (!repacked) {
+ return;
+ }
+ buffer = repacked->first;
+ offset = static_cast<u32>(repacked->second);
+ size = static_cast<u32>((u64{size} + 68) / 69 * 72);
+ stride = 72;
+ }
if (device.IsExtExtendedDynamicStateSupported()) {
scheduler.Record([index, buffer, offset, size, stride](vk::CommandBuffer cmdbuf) {
const VkDeviceSize vk_offset = buffer != VK_NULL_HANDLE ? offset : 0;
@@ -661,6 +694,24 @@ void BufferCacheRuntime::BindVertexBuffers(VideoCommon::HostBindings<Buffer>& bi
}
}
buffer_handles[i] = handle;
+ const u32 binding = bindings.min_index + i;
+ if (handle != VK_NULL_HANDLE && IsVertexRepacked(binding) && bindings.strides[i] == 69 &&
+ bindings.sizes[i] > 0) {
+ const auto repacked = vertex_repack_pass->Assemble(
+ handle, static_cast<u32>(bindings.offsets[i]), static_cast<u32>(bindings.sizes[i]));
+ if (!repacked) {
+ continue;
+ }
+ buffer_handles[i] = repacked->first;
+ bindings.offsets[i] = repacked->second;
+ bindings.sizes[i] = (bindings.sizes[i] + 68) / 69 * 72;
+ bindings.strides[i] = 72;
+ static bool reported = false;
+ if (!reported) {
+ LOG_WARNING(Render_Vulkan, "Experimental GPU vertex repacking active: 69 -> 72 bytes");
+ reported = true;
+ }
+ }
}
const u32 device_max = device.GetMaxVertexInputBindings();
const u32 min_binding = (std::min)(bindings.min_index, device_max);
diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h
index fff1f8164d..ff854445b5 100644
--- a/src/video_core/renderer_vulkan/vk_buffer_cache.h
+++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h
@@ -110,6 +110,12 @@ public:
void TickFrame(Common::SlotVector<Buffer>& slot_buffers) noexcept;
+ u32 SetVertexRepackLayout(const Tegra::Engines::Maxwell3D::Regs& regs, bool dynamic_input);
+
+ bool IsVertexRepacked(u32 binding) const {
+ return (vertex_repack_mask & (1U << binding)) != 0;
+ }
+
u64 CurrentTick();
bool IsFree(u64 tick);
@@ -259,6 +265,8 @@ private:
vk::Buffer null_buffer;
std::unique_ptr<Uint8Pass> uint8_pass;
+ std::unique_ptr<VertexRepackPass> vertex_repack_pass;
+ u32 vertex_repack_mask{};
QuadIndexedPass quad_index_pass;
MultiRangeBufferCache multi_range_buffers;
diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp
index 8cd8abb599..cb8dcde42f 100644
--- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp
+++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp
@@ -26,8 +26,10 @@
#include "video_core/host_shaders/resolve_conditional_render_comp_spv.h"
#include "video_core/host_shaders/vulkan_quad_indexed_comp_spv.h"
#include "video_core/host_shaders/vulkan_uint8_comp_spv.h"
+#include "video_core/host_shaders/vulkan_vertex_repack_comp_spv.h"
#include "video_core/host_shaders/block_linear_unswizzle_3d_bcn_comp_spv.h"
#include "video_core/renderer_vulkan/vk_compute_pass.h"
+#include "video_core/renderer_vulkan/vertex_repack.h"
#include "video_core/surface.h"
#include "video_core/renderer_vulkan/vk_descriptor_pool.h"
#include "video_core/renderer_vulkan/vk_scheduler.h"
@@ -324,6 +326,62 @@ ComputePass::ComputePass(const Device& device_, Scheduler& scheduler, Descriptor
ComputePass::~ComputePass() = default;
+VertexRepackPass::VertexRepackPass(const Device& device_, Scheduler& scheduler_,
+ DescriptorPool& descriptor_pool, StagingBufferPool& staging_pool_,
+ ComputePassDescriptorQueue& descriptor_queue_)
+ : ComputePass(device_, scheduler_, descriptor_pool, INPUT_OUTPUT_DESCRIPTOR_SET_BINDINGS,
+ INPUT_OUTPUT_DESCRIPTOR_UPDATE_TEMPLATE, INPUT_OUTPUT_BANK_INFO,
+ COMPUTE_PUSH_CONSTANT_RANGE<12>, VULKAN_VERTEX_REPACK_COMP_SPV),
+ scheduler{scheduler_}, staging_pool{staging_pool_}, descriptor_queue{descriptor_queue_} {}
+
+std::optional<std::pair<VkBuffer, VkDeviceSize>> VertexRepackPass::Assemble(VkBuffer source,
+ u32 offset, u32 size) {
+ const auto parameters = MakeVertexRepackParameters(
+ offset, size, device.GetStorageBufferAlignment(), device.GetMaxStorageBufferRange(),
+ device.GetMaxComputeWorkGroupCount()[0]);
+ if (!parameters) {
+ LOG_ERROR(Render_Vulkan, "Vertex repack exceeds device limits: offset={}, size={}", offset, size);
+ return std::nullopt;
+ }
+ const u32 num_vertices = parameters->vertices;
+ const u32 output_size = parameters->output_size;
+ const auto dest = staging_pool.Request(output_size, MemoryUsage::DeviceLocal);
+ const u32 base_offset = parameters->source_base;
+ const u32 bias = parameters->source_bias;
+ descriptor_queue.Acquire(scheduler, 2);
+ descriptor_queue.AddBuffer(source, base_offset, parameters->source_range);
+ descriptor_queue.AddBuffer(dest.buffer, dest.offset, output_size);
+ const void* data = descriptor_queue.UpdateData();
+ scheduler.RequestOutsideRenderPassOperationContext();
+ LOG_DEBUG(Render_Vulkan, "Vertex repack queued: source_size={}, output_size={}, groups={}",
+ size, output_size, parameters->dispatch_groups);
+ scheduler.Record([this, data, bias, size, num_vertices,
+ groups = parameters->dispatch_groups](vk::CommandBuffer cmdbuf) {
+ const VkMemoryBarrier read_barrier{
+ .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
+ .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT,
+ .dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
+ };
+ cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
+ VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, read_barrier);
+ const auto set = descriptor_allocator.Commit();
+ device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, data);
+ cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline);
+ cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {});
+ const std::array<u32, 3> constants{bias, size, num_vertices * 18};
+ cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(constants), constants.data());
+ cmdbuf.Dispatch(groups, 1, 1);
+ const VkMemoryBarrier write_barrier{
+ .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
+ .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
+ .dstAccessMask = VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT,
+ };
+ cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
+ VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, 0, write_barrier);
+ });
+ return std::pair<VkBuffer, VkDeviceSize>{dest.buffer, dest.offset};
+}
+
Uint8Pass::Uint8Pass(const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool,
StagingBufferPool& staging_buffer_pool_,
ComputePassDescriptorQueue& compute_pass_descriptor_queue_)
@@ -743,11 +801,11 @@ void ASTCDecoderPass::AssembleInto(const VideoCommon::ImageInfo& info, u32 skip,
cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, image_barrier);
});
- // Each decode ends with the GPU thread waiting for the host GPU to go idle. Measured here
- // (count and time per 5 s); EDEN_NX2_ASTC_FLUSH=1 submits without waiting instead.
+ // Each decode ends with a submit to the host GPU. Measured here
+ // (count and time per 5 s); EDEN_NX2_ASTC_FLUSH=0 waits for the host GPU to go idle instead.
static const bool flush_only = [] {
const char* value = std::getenv("EDEN_NX2_ASTC_FLUSH");
- return value != nullptr && value[0] == '1';
+ return value == nullptr || value[0] != '0';
}();
if (!target.wait) {
return;
diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.h b/src/video_core/renderer_vulkan/vk_compute_pass.h
index 9f2e8e9f1c..27fffbbbb7 100644
--- a/src/video_core/renderer_vulkan/vk_compute_pass.h
+++ b/src/video_core/renderer_vulkan/vk_compute_pass.h
@@ -56,6 +56,20 @@ private:
vk::ShaderModule module;
};
+class VertexRepackPass final : public ComputePass {
+public:
+ VertexRepackPass(const Device& device, Scheduler& scheduler, DescriptorPool& descriptor_pool,
+ StagingBufferPool& staging_pool, ComputePassDescriptorQueue& descriptor_queue);
+ /// Returns nothing when the range exceeds the device limits; the binding is then left as is.
+ std::optional<std::pair<VkBuffer, VkDeviceSize>> Assemble(VkBuffer source, u32 offset,
+ u32 size);
+
+private:
+ Scheduler& scheduler;
+ StagingBufferPool& staging_pool;
+ ComputePassDescriptorQueue& descriptor_queue;
+};
+
class Uint8Pass final : public ComputePass {
public:
explicit Uint8Pass(const Device& device_, Scheduler& scheduler_,
diff --git a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp
index 29356cfcec..1cfe391ba6 100644
--- a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp
+++ b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp
@@ -18,6 +18,7 @@
#include "video_core/renderer_vulkan/vk_pipeline_cache.h"
#include "video_core/renderer_vulkan/vk_scheduler.h"
#include "video_core/renderer_vulkan/vk_update_descriptor.h"
+#include "video_core/shader_handle_program.h"
#include "video_core/shader_notify.h"
#include "video_core/vulkan_common/vulkan_device.h"
#include "video_core/vulkan_common/vulkan_wrapper.h"
@@ -134,6 +135,9 @@ ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk
if (pipeline_statistics) {
pipeline_statistics->Collect(device, *pipeline);
}
+ // Nothing reads the module once the pipeline exists; the driver keeps its SPIR-V while it
+ // lives.
+ spv_module.reset();
std::scoped_lock lock{build_mutex};
is_built = true;
build_condvar.notify_one();
@@ -172,6 +176,20 @@ bool ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute,
const auto& cbufs{qmd.const_buffer_config};
const bool via_header_index{qmd.linked_tsc != 0};
const auto read_handle{[&](const auto& desc, u32 index) {
+ if constexpr (std::is_same_v<decltype(desc), const Shader::TextureDescriptor&>) {
+ if (desc.handle_program != ~0U) {
+ // Read by the shader from a storage buffer: computed for this dispatch.
+ return TexturePair(VideoCommon::EvaluateTextureHandle(
+ info, desc.handle_program, gpu_memory,
+ [&](u32 cbuf) -> GPUVAddr {
+ return cbuf < cbufs.size() &&
+ ((qmd.const_buffer_enable_mask >> cbuf) & 1) != 0
+ ? cbufs[cbuf].Address()
+ : GPUVAddr{};
+ }),
+ via_header_index);
+ }
+ }
ASSERT(((qmd.const_buffer_enable_mask >> desc.cbuf_index) & 1) != 0);
const u32 index_offset{index << desc.size_shift};
const u32 offset{desc.cbuf_offset + index_offset};
diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp
index 94bace432c..fa273f6ef9 100644
--- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp
+++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp
@@ -24,6 +24,7 @@
#include "video_core/renderer_vulkan/vk_scheduler.h"
#include "video_core/renderer_vulkan/vk_texture_cache.h"
#include "video_core/renderer_vulkan/vk_update_descriptor.h"
+#include "video_core/shader_handle_program.h"
#include "video_core/shader_notify.h"
#include "video_core/texture_cache/texture_cache.h"
#include "video_core/vulkan_common/vulkan_device.h"
@@ -54,6 +55,17 @@ using VideoCore::Surface::PixelFormatFromRenderTargetFormat;
constexpr size_t NUM_STAGES = Maxwell::MaxShaderStage;
+// The stage of a program: task and mesh programs sit in the vertex and tessellation slots.
+VkShaderStageFlagBits StageFlag(const Shader::Info& info, VkShaderStageFlagBits slot_stage) {
+ if (info.is_task) {
+ return VK_SHADER_STAGE_TASK_BIT_EXT;
+ }
+ if (info.is_mesh) {
+ return VK_SHADER_STAGE_MESH_BIT_EXT;
+ }
+ return slot_stage;
+}
+
DescriptorLayoutBuilder MakeBuilder(const Device& device, std::span<const Shader::Info> infos) {
DescriptorLayoutBuilder builder{device};
for (size_t index = 0; index < infos.size(); ++index) {
@@ -64,7 +76,7 @@ DescriptorLayoutBuilder MakeBuilder(const Device& device, std::span<const Shader
VK_SHADER_STAGE_GEOMETRY_BIT,
VK_SHADER_STAGE_FRAGMENT_BIT,
};
- builder.Add(infos[index], stages.at(index));
+ builder.Add(infos[index], StageFlag(infos[index], stages.at(index)));
}
return builder;
}
@@ -330,6 +342,11 @@ GraphicsPipeline::GraphicsPipeline(
if (pipeline_statistics) {
pipeline_statistics->Collect(device, *pipeline);
}
+ // Nothing reads the shader modules once the pipeline exists, and the driver keeps a copy
+ // of each one's SPIR-V while it lives (title 1: ~1,500 pipelines, 2-5 modules each).
+ for (vk::ShaderModule& module : spv_modules) {
+ module.reset();
+ }
std::scoped_lock lock{build_mutex};
is_built = true;
@@ -338,12 +355,13 @@ GraphicsPipeline::GraphicsPipeline(
shader_notify->MarkShaderComplete();
}
}};
+ // Before the build: it frees the modules this looks at.
+ configure_func = ConfigureFunc(spv_modules, stage_infos);
if (worker_thread) {
worker_thread->QueueWork(std::move(func));
} else {
func();
}
- configure_func = ConfigureFunc(spv_modules, stage_infos);
}
void GraphicsPipeline::AddTransition(GraphicsPipeline* transition) {
@@ -382,6 +400,19 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) {
}
const auto& cbufs{maxwell3d->state.shader_stages[stage].const_buffers};
const auto read_handle{[&](const auto& desc, u32 index) {
+ if constexpr (std::is_same_v<decltype(desc), const Shader::TextureDescriptor&>) {
+ if (desc.handle_program != ~0U) {
+ // Read by the shader from a storage buffer: computed for this draw.
+ return TexturePair(VideoCommon::EvaluateTextureHandle(
+ info, desc.handle_program, *gpu_memory,
+ [&](u32 cbuf) -> GPUVAddr {
+ return cbuf < cbufs.size() && cbufs[cbuf].enabled
+ ? cbufs[cbuf].address
+ : GPUVAddr{};
+ }),
+ via_header_index);
+ }
+ }
ASSERT(cbufs[desc.cbuf_index].enabled);
const u32 index_offset{index << desc.size_shift};
const u32 offset{desc.cbuf_offset + index_offset};
@@ -1073,6 +1104,19 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) {
}
}
+ const bool is_mesh{IsMeshPipeline()};
+ if (is_mesh) {
+ // Mesh pipelines have no vertex input or input assembly.
+ dynamic_states.erase(std::remove_if(dynamic_states.begin(), dynamic_states.end(),
+ [](VkDynamicState state) {
+ return state == VK_DYNAMIC_STATE_PRIMITIVE_TOPOLOGY_EXT ||
+ state == VK_DYNAMIC_STATE_PRIMITIVE_RESTART_ENABLE_EXT ||
+ state == VK_DYNAMIC_STATE_VERTEX_INPUT_EXT ||
+ state == VK_DYNAMIC_STATE_VERTEX_INPUT_BINDING_STRIDE_EXT ||
+ state == VK_DYNAMIC_STATE_PATCH_CONTROL_POINTS_EXT;
+ }),
+ dynamic_states.end());
+ }
const VkPipelineDynamicStateCreateInfo dynamic_state_ci{
.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
.pNext = nullptr,
@@ -1095,7 +1139,8 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) {
.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
.pNext = nullptr,
.flags = 0,
- .stage = MaxwellToVK::ShaderStage(Shader::StageFromIndex(stage)),
+ .stage = StageFlag(stage_infos[stage],
+ MaxwellToVK::ShaderStage(Shader::StageFromIndex(stage))),
.module = *spv_modules[stage],
.pName = "main",
.pSpecializationInfo = nullptr,
@@ -1115,9 +1160,9 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) {
.flags = flags,
.stageCount = static_cast<u32>(shader_stages.size()),
.pStages = shader_stages.data(),
- .pVertexInputState = &vertex_input_ci,
- .pInputAssemblyState = &input_assembly_ci,
- .pTessellationState = &tessellation_ci,
+ .pVertexInputState = is_mesh ? nullptr : &vertex_input_ci,
+ .pInputAssemblyState = is_mesh ? nullptr : &input_assembly_ci,
+ .pTessellationState = is_mesh ? nullptr : &tessellation_ci,
.pViewportState = &viewport_ci,
.pRasterizationState = &rasterization_ci,
.pMultisampleState = &multisample_ci,
diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h
index b1ac5a3fcc..6d6e2ab12d 100644
--- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h
+++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h
@@ -98,6 +98,11 @@ public:
bool UsesExtendedDynamicState() const noexcept {
return key.state.extended_dynamic_state != 0;
}
+
+ /// Switch 2 task/mesh pipeline: drawn with vkCmdDrawMeshTasksEXT.
+ [[nodiscard]] bool IsMeshPipeline() const noexcept {
+ return stage_infos[0].is_task || stage_infos[0].is_mesh;
+ }
GraphicsPipeline& operator=(GraphicsPipeline&&) noexcept = delete;
GraphicsPipeline(GraphicsPipeline&&) noexcept = delete;
@@ -119,6 +124,21 @@ public:
: nullptr;
}
+ /// The frame the pipeline cache last handed this pipeline out (evicts long unused ones).
+ void MarkUsed(u64 frame) noexcept {
+ last_use_frame = frame;
+ }
+
+ [[nodiscard]] u64 LastUse() const noexcept {
+ return last_use_frame;
+ }
+
+ /// Forgets the transitions, which may point to pipelines the cache evicted.
+ void ClearTransitions() noexcept {
+ transition_keys.clear();
+ transitions.clear();
+ }
+
[[nodiscard]] bool IsBuilt() const noexcept {
return is_built.load(std::memory_order::relaxed);
}
@@ -159,6 +179,7 @@ private:
std::vector<GraphicsPipelineCacheKey> transition_keys;
std::vector<GraphicsPipeline*> transitions;
+ u64 last_use_frame{};
std::array<vk::ShaderModule, NUM_STAGES> spv_modules;
diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
index 05511c10e1..ff8e84b019 100644
--- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
@@ -170,6 +170,14 @@ Shader::RuntimeInfo MakeRuntimeInfo(std::span<const Shader::IR::Program> program
} else {
info.previous_stage_stores.mask.set();
}
+ if (previous_program && previous_program->stage == Shader::Stage::Mesh) {
+ for (const u32 line : previous_program->mesh.primitive_lines) {
+ const auto attribute{static_cast<Shader::IR::Attribute>(line)};
+ if (Shader::IR::IsGeneric(attribute)) {
+ info.per_primitive_generics.set(Shader::IR::GenericAttributeIndex(attribute));
+ }
+ }
+ }
const Shader::Stage stage{program.stage};
const bool has_geometry{key.unique_hashes[4] != 0 && !programs[4].is_geometry_passthrough};
const bool gl_ndc{key.state.ndc_minus_one_to_one != 0};
@@ -230,6 +238,9 @@ Shader::RuntimeInfo MakeRuntimeInfo(std::span<const Shader::IR::Program> program
return Shader::TessSpacing::Equal;
}();
break;
+ case Shader::Stage::Mesh:
+ info.convert_depth_mode = gl_ndc;
+ break;
case Shader::Stage::Geometry:
if (program.output_topology == Shader::OutputTopology::PointList) {
info.fixed_state_point_size = point_size;
@@ -345,6 +356,22 @@ bool GraphicsPipelineCacheKey::operator==(const GraphicsPipelineCacheKey& rhs) c
return std::memcmp(&rhs, this, Size()) == 0;
}
+namespace {
+// NVIDIA's and AMD's drivers keep a disk cache of built pipelines of their own. Eden's copy on top
+// only kept every pipeline the title ever built in host memory (title 1: a 245 MB file, ~0.35 GB
+// of RAM for the session) and copied it whole at each save, which stalled frames; pipelines built
+// no slower without it.
+bool DriverCachesPipelines(const Device& device) {
+ switch (device.GetDriverID()) {
+ case VK_DRIVER_ID_NVIDIA_PROPRIETARY:
+ case VK_DRIVER_ID_AMD_PROPRIETARY:
+ return true;
+ default:
+ return false;
+ }
+}
+} // Anonymous namespace
+
PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
const Device& device_, Scheduler& scheduler_,
DescriptorPool& descriptor_pool_,
@@ -358,7 +385,8 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
render_pass_cache{render_pass_cache_}, buffer_cache{buffer_cache_},
texture_cache{texture_cache_}, shader_notify{shader_notify_},
use_asynchronous_shaders{Settings::values.use_asynchronous_shaders.GetValue()},
- use_vulkan_pipeline_cache{Settings::values.use_vulkan_driver_pipeline_cache.GetValue()},
+ use_vulkan_pipeline_cache{Settings::values.use_vulkan_driver_pipeline_cache.GetValue() &&
+ !DriverCachesPipelines(device_)},
workers(device.HasBrokenParallelShaderCompiling() ? 1ULL : GetTotalPipelineWorkers(),
"VkPipelineBuilder", {}, Common::ThreadPlacement::Background),
serialization_thread(1, "VkPipelineSerialization", {},
@@ -378,7 +406,9 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
Shader::Stage::TessellationEval) |
subgroup_stage_bit(VK_SHADER_STAGE_GEOMETRY_BIT, Shader::Stage::Geometry) |
subgroup_stage_bit(VK_SHADER_STAGE_FRAGMENT_BIT, Shader::Stage::Fragment) |
- subgroup_stage_bit(VK_SHADER_STAGE_COMPUTE_BIT, Shader::Stage::Compute)};
+ subgroup_stage_bit(VK_SHADER_STAGE_COMPUTE_BIT, Shader::Stage::Compute) |
+ subgroup_stage_bit(VK_SHADER_STAGE_TASK_BIT_EXT, Shader::Stage::Task) |
+ subgroup_stage_bit(VK_SHADER_STAGE_MESH_BIT_EXT, Shader::Stage::Mesh)};
if (driver_id == VK_DRIVER_ID_MESA_TURNIP) {
supported_subgroup_stages &= ~(1u << static_cast<u32>(Shader::Stage::Geometry));
}
@@ -563,13 +593,96 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_,
device.SupportsTransformFeedbackProvokingVertexPreservation();
}
+namespace {
+// <title>/vulkan_used.bin: the graphics pipelines recent sessions used, as key hash + sessions
+// since the last use.
+constexpr u32 UsedPipelinesMagic = 0x31505545; // "EUP1"
+// Pipelines the last two sessions used are built at boot. Each holds ~0.45 MB of driver memory:
+// title 1 built 4,800 from eight sessions, 1.5 GB more than building them on first use (but that
+// stalled frames 4x as often), while one session of its title screen uses ~1,500.
+constexpr u8 UsedPipelinesMaxAge = 2;
+
+std::unordered_map<u64, u8> LoadUsedPipelines(const std::filesystem::path& path) {
+ std::unordered_map<u64, u8> used;
+ std::ifstream file{path, std::ios::binary};
+ u32 magic{}, count{};
+ if (!file.read(reinterpret_cast<char*>(&magic), sizeof(magic)) || magic != UsedPipelinesMagic ||
+ !file.read(reinterpret_cast<char*>(&count), sizeof(count))) {
+ return used;
+ }
+ used.reserve(count);
+ for (u32 i = 0; i < count; ++i) {
+ u64 hash{};
+ u8 age{};
+ if (!file.read(reinterpret_cast<char*>(&hash), sizeof(hash)) ||
+ !file.read(reinterpret_cast<char*>(&age), sizeof(age))) {
+ break;
+ }
+ if (age < UsedPipelinesMaxAge) { // lists saved with a longer window
+ used.emplace(hash, age);
+ }
+ }
+ return used;
+}
+
+// Without a list of the pipelines recent sessions used (the first boot with this cache, or a session
+// that ended before saving it), this many cached graphics pipelines at most are built at boot and the
+// rest when first needed. A cache that grew over many builds held 48k pipelines, ~21 GB of driver
+// memory built before the game started; the list is only saved at exit or after two minutes, so a
+// boot that ran out of memory never saved one and the next boot did the same.
+constexpr size_t MaxPipelinesWithoutUsedList = 3000;
+
+// EDEN_NX2_PRELOAD_ALL_PIPELINES=1 builds every cached pipeline at boot, as before.
+bool PreloadAllPipelines() {
+ static const bool all = [] {
+ const char* value = std::getenv("EDEN_NX2_PRELOAD_ALL_PIPELINES");
+ return value != nullptr && value[0] == '1';
+ }();
+ return all;
+}
+} // Anonymous namespace
+
PipelineCache::~PipelineCache() {
+ SaveUsedPipelines();
if (use_vulkan_pipeline_cache && !vulkan_pipeline_cache_filename.empty()) {
SerializeVulkanPipelineCache(vulkan_pipeline_cache_filename, vulkan_pipeline_cache,
CACHE_VERSION);
}
}
+void PipelineCache::SaveUsedPipelines() {
+ used_pipelines_saved = std::chrono::steady_clock::now();
+ // A session too short to say what the game uses (a boot to the menu) keeps the old list.
+ constexpr size_t MinimumUsed = 64;
+ if (used_pipelines_filename.empty() ||
+ used_this_session.size() + used_and_evicted.size() < MinimumUsed) {
+ return;
+ }
+ std::unordered_map<u64, u8> next;
+ next.reserve(used_pipelines.size() + used_this_session.size());
+ for (const auto& [key, pipeline] : graphics_cache) {
+ if (used_this_session.contains(pipeline.get())) {
+ next.emplace(key.Hash(), u8{0});
+ }
+ }
+ for (const u64 hash : used_and_evicted) {
+ next.emplace(hash, u8{0});
+ }
+ for (const auto& [hash, age] : used_pipelines) {
+ if (age + 1 < UsedPipelinesMaxAge) {
+ next.try_emplace(hash, static_cast<u8>(age + 1));
+ }
+ }
+ std::ofstream file{used_pipelines_filename, std::ios::binary | std::ios::trunc};
+ const u32 count = static_cast<u32>(next.size());
+ file.write(reinterpret_cast<const char*>(&UsedPipelinesMagic), sizeof(UsedPipelinesMagic));
+ file.write(reinterpret_cast<const char*>(&count), sizeof(count));
+ for (const auto& [hash, age] : next) {
+ file.write(reinterpret_cast<const char*>(&hash), sizeof(hash));
+ file.write(reinterpret_cast<const char*>(&age), sizeof(age));
+ }
+}
+
GraphicsPipeline* PipelineCache::CurrentGraphicsPipeline() {
if (!RefreshStages(graphics_key.unique_hashes)) {
@@ -582,6 +695,7 @@ GraphicsPipeline* PipelineCache::CurrentGraphicsPipeline() {
GraphicsPipeline* const next{current_pipeline->Next(graphics_key)};
if (next) {
current_pipeline = next;
+ current_pipeline->MarkUsed(frame_count);
return BuiltPipeline(current_pipeline);
}
}
@@ -629,6 +743,11 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading
return;
}
pipeline_cache_filename = base_dir / "vulkan.bin";
+ used_pipelines_filename = base_dir / "vulkan_used.bin";
+ used_pipelines = LoadUsedPipelines(used_pipelines_filename);
+ used_pipelines_saved = std::chrono::steady_clock::now();
+ size_t skipped_pipelines = 0;
+ size_t queued_graphics = 0;
if (use_vulkan_pipeline_cache) {
vulkan_pipeline_cache_filename = base_dir / "vulkan_pipelines.bin";
@@ -669,6 +788,15 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading
GraphicsPipelineCacheKey key;
file.read(reinterpret_cast<char*>(&key), sizeof(key));
+ // Task/mesh pipelines are not stored (their mesh registers are not in the format); skip
+ // any that were.
+ if (std::ranges::any_of(envs, [](const FileEnvironment& env) {
+ return env.ShaderStage() == Shader::Stage::Task ||
+ env.ShaderStage() == Shader::Stage::Mesh;
+ })) {
+ return;
+ }
+
if ((key.state.extended_dynamic_state != 0) !=
dynamic_features.has_extended_dynamic_state ||
(key.state.extended_dynamic_state_2 != 0) !=
@@ -697,6 +825,16 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading
return;
}
+ const u64 hash = key.Hash();
+ disk_pipelines.insert(hash);
+ if (!PreloadAllPipelines() &&
+ (used_pipelines.empty() ? queued_graphics >= MaxPipelinesWithoutUsedList
+ : !used_pipelines.contains(hash))) {
+ ++skipped_pipelines;
+ return;
+ }
+ ++queued_graphics;
+
workers.QueueWork([this, key, envs_ = std::move(envs), &state, &callback]() mutable {
ShaderPools pools;
boost::container::static_vector<Shader::Environment*, 5> env_ptrs;
@@ -721,6 +859,11 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading
load_graphics);
LOG_INFO(Render_Vulkan, "Total Pipeline Count: {}", state.total);
+ if (skipped_pipelines != 0) {
+ LOG_INFO(Render_Vulkan,
+ "{} cached graphics pipelines are built when first needed",
+ skipped_pipelines);
+ }
std::unique_lock lock{state.mutex};
callback(VideoCore::LoadCallbackStage::Build, 0, state.total);
@@ -837,13 +980,66 @@ GraphicsPipeline* PipelineCache::CurrentGraphicsPipelineSlowPath() {
if (!pipeline) {
return nullptr;
}
+ if (used_this_session.insert(pipeline.get()).second &&
+ std::chrono::steady_clock::now() - used_pipelines_saved > std::chrono::minutes{2}) {
+ SaveUsedPipelines();
+ }
if (current_pipeline) {
current_pipeline->AddTransition(pipeline.get());
}
current_pipeline = pipeline.get();
+ current_pipeline->MarkUsed(frame_count);
return BuiltPipeline(current_pipeline);
}
+void PipelineCache::TickFrame() {
+ ++frame_count;
+ while (!evicted_pipelines.empty() && scheduler.IsFree(evicted_pipelines.front().first)) {
+ evicted_pipelines.pop_front();
+ }
+ constexpr u64 EvictCheckFrames = 600;
+ if (frame_count % EvictCheckFrames == 0) {
+ EvictUnusedPipelines();
+ }
+}
+
+void PipelineCache::EvictUnusedPipelines() {
+ // Each built pipeline holds ~0.45 MB of driver memory and none was ever destroyed, so it grew
+ // with every place a session went (title 1: 1,500 on its title screen alone). One unused for
+ // this long is destroyed; it takes 2-3 ms to build again from the driver's pipeline cache.
+ constexpr u64 EvictAfterFrames = 9000;
+ if (frame_count <= EvictAfterFrames) {
+ return;
+ }
+ const u64 cutoff = frame_count - EvictAfterFrames;
+ const u64 tick = scheduler.CurrentTick();
+ size_t evicted = 0;
+ for (auto it = graphics_cache.begin(); it != graphics_cache.end();) {
+ GraphicsPipeline* const pipeline = it->second.get();
+ if (pipeline == nullptr || pipeline == current_pipeline || !pipeline->IsBuilt() ||
+ pipeline->LastUse() >= cutoff) {
+ ++it;
+ continue;
+ }
+ if (used_this_session.erase(pipeline) != 0) {
+ used_and_evicted.insert(it->first.Hash());
+ }
+ evicted_pipelines.emplace_back(tick, std::move(it->second));
+ it = graphics_cache.erase(it);
+ ++evicted;
+ }
+ if (evicted == 0) {
+ return;
+ }
+ for (auto& [key, pipeline] : graphics_cache) {
+ if (pipeline) {
+ pipeline->ClearTransitions();
+ }
+ }
+ LOG_INFO(Render_Vulkan, "Evicted {} graphics pipelines unused for {} frames, {} left", evicted,
+ EvictAfterFrames, graphics_cache.size());
+}
+
GraphicsPipeline* PipelineCache::BuiltPipeline(GraphicsPipeline* pipeline) const noexcept {
if (pipeline->IsBuilt()) {
return pipeline;
@@ -985,7 +1181,12 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline(
++env_index;
const u32 cfg_offset{static_cast<u32>(env.StartAddress() + sizeof(Shader::ProgramHeader))};
- Shader::Maxwell::Flow::CFG cfg(env, pools.flow_block, cfg_offset, index == 0);
+ try {
+ // Walks the program for the dump; the one that failed may fail here again, and an
+ // exception leaving this handler would end the process.
+ Shader::Maxwell::Flow::CFG cfg(env, pools.flow_block, cfg_offset, index == 0);
+ } catch (const Shader::Exception&) {
+ }
env.Dump(hash, key.unique_hashes[index]);
}
LOG_ERROR(Render_Vulkan, "{}", exception.what());
@@ -1033,6 +1234,18 @@ std::unique_ptr<GraphicsPipeline> PipelineCache::CreateGraphicsPipeline() {
if (!pipeline || pipeline_cache_filename.empty()) {
return pipeline;
}
+ if (pipeline->IsMeshPipeline()) {
+ // Not stored: task and mesh programs are translated with the mesh registers, which the
+ // cache format does not carry. The few of them rebuild quickly from the driver cache.
+ QueueVulkanPipelineCacheFlush();
+ return pipeline;
+ }
+ if (disk_pipelines.contains(graphics_key.Hash())) {
+ // In the disk cache already, only not built at boot.
+ QueueVulkanPipelineCacheFlush();
+ return pipeline;
+ }
+ disk_pipelines.insert(graphics_key.Hash());
serialization_thread.QueueWork([this, key = graphics_key, envs = std::move(environments.envs)] {
boost::container::static_vector<const GenericEnvironment*, Maxwell::MaxShaderProgram>
env_ptrs;
@@ -1055,7 +1268,8 @@ std::unique_ptr<ComputePipeline> PipelineCache::CreateComputePipeline(
};
const GPUVAddr program_base{kepler_compute->regs.code_loc.Address()};
const auto& qmd{kepler_compute->launch_description};
- ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start};
+ ComputeEnvironment env{*kepler_compute, *gpu_memory, program_base, qmd.program_start,
+ shader->lowered_native};
env.SetCachedSize(shader->size_bytes);
main_pools.ReleaseContents();
diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.h b/src/video_core/renderer_vulkan/vk_pipeline_cache.h
index 71dfb46e3d..66fa615ea5 100644
--- a/src/video_core/renderer_vulkan/vk_pipeline_cache.h
+++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.h
@@ -10,7 +10,10 @@
#include <atomic>
#include <chrono>
#include <cstddef>
+#include <deque>
#include <filesystem>
+#include <unordered_map>
+#include <unordered_set>
#include <memory>
#include <type_traits>
#include "common/container/unordered_map.h"
@@ -114,6 +117,8 @@ public:
[[nodiscard]] GraphicsPipeline* CurrentGraphicsPipeline();
+ void TickFrame();
+
[[nodiscard]] ComputePipeline* CurrentComputePipeline();
void LoadDiskResources(u64 title_id, std::stop_token stop_loading,
@@ -163,6 +168,12 @@ private:
GraphicsPipelineCacheKey graphics_key{};
GraphicsPipeline* current_pipeline{};
+ /// Destroys graphics pipelines unused for a while, once the GPU is done with them.
+ void EvictUnusedPipelines();
+ u64 frame_count{};
+ std::deque<std::pair<u64, std::unique_ptr<GraphicsPipeline>>> evicted_pipelines; ///< (tick, pipeline)
+ std::unordered_set<u64> used_and_evicted; ///< Key hashes of pipelines used this session, evicted since
+
::Common::unordered_map<ComputePipelineCacheKey, std::unique_ptr<ComputePipeline>> compute_cache;
::Common::unordered_map<GraphicsPipelineCacheKey, std::unique_ptr<GraphicsPipeline>> graphics_cache;
@@ -173,6 +184,16 @@ private:
std::filesystem::path pipeline_cache_filename;
+ /// Key hashes of the graphics pipelines in the disk cache. Only those a recent session used
+ /// (used_pipelines) are built at boot; the rest are built when first needed, and not stored
+ /// again. Each built pipeline holds driver memory (title 1: 6.5k cached, a quarter used).
+ std::unordered_set<u64> disk_pipelines;
+ std::unordered_map<u64, u8> used_pipelines; ///< Key hash -> sessions since it was last used
+ std::unordered_set<const GraphicsPipeline*> used_this_session;
+ std::filesystem::path used_pipelines_filename;
+ std::chrono::steady_clock::time_point used_pipelines_saved{};
+ void SaveUsedPipelines();
+
std::filesystem::path vulkan_pipeline_cache_filename;
vk::PipelineCache vulkan_pipeline_cache;
size_t pipelines_since_flush{};
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index cb31a0e7e4..d3dbba6163 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -9,6 +9,7 @@
#include <memory>
#include <cstdio>
#include <cstdlib>
+#include <filesystem>
#include <tuple>
#include <set>
#include <fstream>
@@ -279,9 +280,37 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) {
report_failure("no pipeline");
return;
}
+ // Task/mesh pipelines: the draw's vertex count is the number of task workgroups.
+ mesh_draw = pipeline->IsMeshPipeline();
+ // EDEN_NX2_NO_MESH=1: skip task/mesh draws as before they were supported (A/B comparisons).
+ // EDEN_NX2_NO_MESH_FILE=<path>: skip them while that file exists (checked every 1024 draws),
+ // to compare the same scene with and without them in one run.
+ static const bool no_mesh_always{std::getenv("EDEN_NX2_NO_MESH") != nullptr};
+ static const char* const no_mesh_file{std::getenv("EDEN_NX2_NO_MESH_FILE")};
+ static bool no_mesh{no_mesh_always};
+ static u32 no_mesh_check{};
+ if (no_mesh_file && (no_mesh_check++ % 1024) == 0) {
+ no_mesh = std::filesystem::exists(no_mesh_file);
+ }
+ if (mesh_draw && (no_mesh || !device.IsExtMeshShaderSupported())) {
+ report_failure(no_mesh ? "EDEN_NX2_NO_MESH" : "no VK_EXT_mesh_shader");
+ return;
+ }
std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex};
// update engine as channel may be different.
pipeline->SetEngine(maxwell3d, gpu_memory);
+ const u32 repack_dirty = buffer_cache_runtime.SetVertexRepackLayout(
+ maxwell3d->regs, pipeline->HasDynamicVertexInput());
+ if (repack_dirty) {
+ auto& flags = maxwell3d->dirty.flags;
+ flags[VideoCommon::Dirty::VertexBuffers] = true;
+ flags[Dirty::VertexInput] = true;
+ for (u32 binding = 0; binding < 32; ++binding) {
+ if (repack_dirty & (1U << binding)) {
+ flags[VideoCommon::Dirty::VertexBuffer0 + binding] = true;
+ }
+ }
+ }
if (Profile::Scope scope{Profile::Zone::Configure}; !pipeline->Configure(is_indexed)) {
report_failure("configure failed");
return;
@@ -541,6 +570,12 @@ void RasterizerVulkan::Draw(bool is_indexed, u32 instance_count) {
const u32 num_instances{instance_count};
const DrawParams draw_params{MakeDrawParams(draw_state, num_instances, is_indexed)};
+ if (mesh_draw) {
+ scheduler.Record([groups = draw_params.num_vertices](vk::CommandBuffer cmdbuf) {
+ cmdbuf.DrawMeshTasksEXT(groups, 1, 1);
+ });
+ return;
+ }
scheduler.Record([draw_params](vk::CommandBuffer cmdbuf) {
if (draw_params.is_indexed) {
cmdbuf.DrawIndexed(draw_params.num_vertices, draw_params.num_instances,
@@ -573,6 +608,14 @@ void RasterizerVulkan::DrawIndirect() {
const auto& params = maxwell3d->draw_manager.indirect_state;
buffer_cache.SetDrawIndirect(&params);
PrepareDraw(params.is_indexed, [this, &params] {
+ if (mesh_draw) {
+ // Indirect task/mesh draws would need vkCmdDrawMeshTasksIndirectEXT's layout.
+ static bool logged{};
+ if (!std::exchange(logged, true)) {
+ LOG_WARNING(Render_Vulkan, "Indirect task/mesh draw skipped");
+ }
+ return;
+ }
const auto indirect_buffer = buffer_cache.GetDrawIndirectBuffer();
const auto& buffer = indirect_buffer.first;
const auto& offset = indirect_buffer.second;
@@ -1200,6 +1243,7 @@ void RasterizerVulkan::FlushCommands() {
void RasterizerVulkan::TickFrame() {
draw_counter = 0;
+ pipeline_cache.TickFrame();
guest_descriptor_queue.TickFrame();
compute_pass_descriptor_queue.TickFrame();
descriptor_buffer_ring.TickFrame();
@@ -1231,6 +1275,14 @@ bool RasterizerVulkan::AccelerateSurfaceCopy(const Tegra::Engines::Fermi2D::Surf
return texture_cache.BlitImage(dst, src, copy_config);
}
+bool RasterizerVulkan::AccelerateTemporalUpscale(
+ const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& dst,
+ const Tegra::Engines::Fermi2D::Surface& motion_vectors, std::array<f32, 2> jitter,
+ std::array<f32, 2> motion_scale) {
+ std::scoped_lock lock{texture_cache.mutex};
+ return texture_cache.TemporalUpscale(dst, src, motion_vectors, jitter, motion_scale);
+}
+
Tegra::Engines::AccelerateDMAInterface& RasterizerVulkan::AccessAccelerateDMA() {
return accelerate_dma;
}
@@ -2261,7 +2313,8 @@ void RasterizerVulkan::UpdateVertexInput(Tegra::Engines::Maxwell3D::Regs& regs)
.location = index,
.binding = binding,
.format = MaxwellToVK::VertexFormat(device, attribute.type, attribute.size),
- .offset = attribute.offset,
+ .offset = attribute.offset +
+ (buffer_cache_runtime.IsVertexRepacked(binding) && attribute.offset >= 65 ? 3 : 0),
});
}
@@ -2272,7 +2325,7 @@ void RasterizerVulkan::UpdateVertexInput(Tegra::Engines::Maxwell3D::Regs& regs)
.sType = VK_STRUCTURE_TYPE_VERTEX_INPUT_BINDING_DESCRIPTION_2_EXT,
.pNext = nullptr,
.binding = binding,
- .stride = input_binding.stride,
+ .stride = buffer_cache_runtime.IsVertexRepacked(binding) ? 72U : input_binding.stride.Value(),
.inputRate = is_instanced ? VK_VERTEX_INPUT_RATE_INSTANCE : VK_VERTEX_INPUT_RATE_VERTEX,
.divisor = is_instanced ? input_binding.frequency : 1,
});
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h
index ae9929ca0b..fc282fbe8d 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.h
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.h
@@ -128,6 +128,11 @@ public:
bool AccelerateSurfaceCopy(const Tegra::Engines::Fermi2D::Surface& src,
const Tegra::Engines::Fermi2D::Surface& dst,
const Tegra::Engines::Fermi2D::Config& copy_config) override;
+ bool AccelerateTemporalUpscale(const Tegra::Engines::Fermi2D::Surface& src,
+ const Tegra::Engines::Fermi2D::Surface& dst,
+ const Tegra::Engines::Fermi2D::Surface& motion_vectors,
+ std::array<f32, 2> jitter,
+ std::array<f32, 2> motion_scale) override;
Tegra::Engines::AccelerateDMAInterface& AccessAccelerateDMA() override;
void AccelerateInlineToMemory(GPUVAddr address, size_t copy_size,
std::span<const u8> memory) override;
@@ -230,6 +235,7 @@ private:
boost::container::static_vector<VkSampler, MAX_TEXTURES> sampler_handles;
u32 draw_counter = 0;
+ bool mesh_draw{}; ///< the draw being prepared uses a task/mesh pipeline
};
} // namespace Vulkan
diff --git a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp
index 7b6571d33d..98787b9add 100644
--- a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp
+++ b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp
@@ -36,11 +36,25 @@ constexpr VkDeviceSize MAX_ALIGNMENT = 256;
#if defined(__FreeBSD__)
constexpr VkDeviceSize MAX_STREAM_BUFFER_SIZE = 128_MiB;
#else
-constexpr VkDeviceSize MAX_STREAM_BUFFER_SIZE = 256_MiB;
+// 128 MiB on Windows too: 256 MiB sat mostly unused in device memory on Switch 2 titles.
+constexpr VkDeviceSize MAX_STREAM_BUFFER_SIZE = 128_MiB;
#endif
size_t GetStreamBufferSize(const Device& device) {
if (!device.HasDebuggingToolAttached()) {
+ if (device.IsIntegrated()) {
+ // An integrated GPU's device-local host-visible heap can be no bigger than the stream
+ // buffer (Radeon 780M: 256 MiB, 243 MiB budget), which then sat over that budget.
+ // Leave a quarter of it free.
+ VkDeviceSize heap_size = 0;
+ ForEachDeviceLocalHostVisibleHeap(
+ device, [&heap_size](size_t, VkMemoryHeap& heap) {
+ heap_size = (std::max)(heap_size, heap.size);
+ });
+ if (heap_size != 0 && heap_size <= MAX_STREAM_BUFFER_SIZE) {
+ return Common::AlignUp(heap_size * 3 / 4, MAX_ALIGNMENT);
+ }
+ }
return MAX_STREAM_BUFFER_SIZE;
}
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
index ddd55de690..30981fe413 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp
@@ -244,6 +244,35 @@ enum class SparseTilesMode {
return size >= (32U << 20) && Tegra::Nx2InSparseRegion(gpu_addr, size);
}
+// Vulkan's standard sparse block shape for single-sampled 2D images (64 KiB blocks), in texel
+// blocks, for a texel block of `block_bytes`. Only valid when the device reports
+// residencyStandard2DBlockShape.
+[[nodiscard]] bool StandardSparseBlockShape(const Device& device, u64 block_bytes,
+ VkExtent3D& shape) {
+ if (!device.GetPhysical().GetProperties().sparseProperties.residencyStandard2DBlockShape) {
+ return false;
+ }
+ switch (block_bytes) {
+ case 1:
+ shape = {256, 256, 1};
+ return true;
+ case 2:
+ shape = {256, 128, 1};
+ return true;
+ case 4:
+ shape = {128, 128, 1};
+ return true;
+ case 8:
+ shape = {128, 64, 1};
+ return true;
+ case 16:
+ shape = {64, 64, 1};
+ return true;
+ default:
+ return false;
+ }
+}
+
// The most device memory one SparseLayers image binds: 768 MiB, 384 MiB on GPUs with less than
// 7 GiB (title 1's BC6H arrays are 85 MB a layer).
[[nodiscard]] VkDeviceSize SparseLayersCap(const Device& device) {
@@ -304,6 +333,20 @@ constexpr std::array<VkFormat, 2> OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM
info.size.depth == 1;
}
+// The most a trimmed ASTC array keeps on the host: 512 MiB, 256 MiB on GPUs with less than 7 GiB
+// (title 1's two terrain arrays took 724 MiB of a 6 GB card next to ~1.2 GB of other sparse terrain
+// textures). EDEN_NX2_ASTC_ARRAY_MIB=<MiB> sets it.
+[[nodiscard]] u64 TrimmedAstcArrayLimit(const Device& device) {
+ static const u64 forced = [] {
+ const char* value = std::getenv("EDEN_NX2_ASTC_ARRAY_MIB");
+ return value != nullptr ? std::strtoull(value, nullptr, 10) << 20 : 0ULL;
+ }();
+ if (forced != 0) {
+ return forced;
+ }
+ return device.GetDeviceLocalMemory() < (u64{7} << 30) ? (u64{256} << 20) : (u64{512} << 20);
+}
+
// The host shape of an image whose decoded copy would not fit in VRAM: a GPU-decoded ASTC array
// over 1 GiB as RGBA8 (title 1's 8192x8192x68 ASTC 8x8 virtual textures are 24 GB each) keeps only
// its mip levels from `level_offset` on, so host mip 0 is guest mip `level_offset`. Normalized
@@ -324,8 +367,9 @@ constexpr std::array<VkFormat, 2> OpaqueAstcViewFormats{VK_FORMAT_BC1_RGBA_UNORM
if (decoded_bytes(0) <= (u64{1} << 30)) {
return info;
}
+ const u64 limit = TrimmedAstcArrayLimit(device);
while (level_offset + 1 < static_cast<u32>(info.resources.levels) &&
- decoded_bytes(level_offset) > (u64{512} << 20)) {
+ decoded_bytes(level_offset) > limit) {
++level_offset;
}
ImageInfo host = info;
@@ -1491,6 +1535,154 @@ void TextureCacheRuntime::ReinterpretImage(Image& dst, Image& src,
});
}
+bool TextureCacheRuntime::TemporalUpscale(Framebuffer* dst_framebuffer, ImageView& dst,
+ ImageView& input_color, ImageView& motion_vectors,
+ std::array<f32, 2> jitter,
+ std::array<f32, 2> motion_scale) {
+ if (dst.Samples() != VK_SAMPLE_COUNT_1_BIT || ImageAspectMask(dst.format) != VK_IMAGE_ASPECT_COLOR_BIT) {
+ return false;
+ }
+ const VkFormat format =
+ MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, false, dst.format).format;
+ const VkExtent2D extent{dst.size.width, dst.size.height};
+ const VkImage output = dst.ImageHandle();
+ bool reset = false;
+ if (!temporal_history.image || temporal_history.format != format ||
+ temporal_history.extent.width != extent.width ||
+ temporal_history.extent.height != extent.height) {
+ if (temporal_history.image) {
+ scheduler.Finish(); // the old history may still be sampled
+ }
+ temporal_history.image = memory_allocator.CreateImage(VkImageCreateInfo{
+ .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
+ .pNext = nullptr,
+ .flags = 0,
+ .imageType = VK_IMAGE_TYPE_2D,
+ .format = format,
+ .extent = {extent.width, extent.height, 1},
+ .mipLevels = 1,
+ .arrayLayers = 1,
+ .samples = VK_SAMPLE_COUNT_1_BIT,
+ .tiling = VK_IMAGE_TILING_OPTIMAL,
+ .usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT,
+ .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
+ .queueFamilyIndexCount = 0,
+ .pQueueFamilyIndices = nullptr,
+ .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
+ });
+ temporal_history.view = device.GetLogical().CreateImageView(VkImageViewCreateInfo{
+ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
+ .pNext = nullptr,
+ .flags = 0,
+ .image = *temporal_history.image,
+ .viewType = VK_IMAGE_VIEW_TYPE_2D,
+ .format = format,
+ .components{},
+ .subresourceRange{
+ .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+ .baseMipLevel = 0,
+ .levelCount = 1,
+ .baseArrayLayer = 0,
+ .layerCount = 1,
+ },
+ });
+ temporal_history.format = format;
+ temporal_history.extent = extent;
+ reset = true;
+ LOG_INFO(Render_Vulkan, "Temporal upscale into {}x{} (format {})", extent.width, extent.height,
+ static_cast<u32>(format));
+ }
+
+ // The history holds this pass's previous result, copied right after drawing it: the game may
+ // draw into its DLSS output afterwards (post-processing), which must not feed back. The history
+ // stays in the GENERAL layout, which the blit helper's descriptors use.
+ const VkImage history = *temporal_history.image;
+ const VkImageSubresourceLayers output_layers{
+ .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+ .mipLevel = static_cast<u32>(dst.range.base.level),
+ .baseArrayLayer = static_cast<u32>(dst.range.base.layer),
+ .layerCount = 1,
+ };
+ static constexpr VkImageSubresourceLayers history_layers{
+ .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+ .mipLevel = 0,
+ .baseArrayLayer = 0,
+ .layerCount = 1,
+ };
+ const auto barrier = [](VkImage image, VkAccessFlags src_access, VkAccessFlags dst_access,
+ VkImageLayout old_layout, const VkImageSubresourceLayers& layers) {
+ return VkImageMemoryBarrier{
+ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+ .pNext = nullptr,
+ .srcAccessMask = src_access,
+ .dstAccessMask = dst_access,
+ .oldLayout = old_layout,
+ .newLayout = VK_IMAGE_LAYOUT_GENERAL,
+ .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+ .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+ .image = image,
+ .subresourceRange{
+ .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+ .baseMipLevel = layers.mipLevel,
+ .levelCount = 1,
+ .baseArrayLayer = layers.baseArrayLayer,
+ .layerCount = 1,
+ },
+ };
+ };
+ if (reset) {
+ // A new history image has no contents yet; the first frame does not read it.
+ scheduler.RequestOutsideRenderPassOperationContext();
+ scheduler.Record([history, barrier](vk::CommandBuffer cmdbuf) {
+ cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
+ 0, barrier(history, 0, VK_ACCESS_SHADER_READ_BIT,
+ VK_IMAGE_LAYOUT_UNDEFINED, history_layers));
+ });
+ }
+ const Region2D dst_region{
+ .start = {0, 0},
+ .end = {static_cast<s32>(extent.width), static_cast<s32>(extent.height)},
+ };
+ // Each frame weighs at most 0.1 against the history (scaled by how close its samples land to
+ // the pixel; the neighbourhood clip limits what older samples contribute).
+ constexpr f32 current_weight = 0.1f;
+ blit_image_helper.TemporalUpscale(dst_framebuffer, input_color, motion_vectors,
+ *temporal_history.view, dst_region, jitter, motion_scale,
+ reset ? 1.0f : current_weight);
+ scheduler.RequestOutsideRenderPassOperationContext();
+ scheduler.Record([output, history, extent, output_layers, barrier](vk::CommandBuffer cmdbuf) {
+ constexpr VkAccessFlags written = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
+ const std::array before{
+ barrier(output, written, VK_ACCESS_TRANSFER_READ_BIT, VK_IMAGE_LAYOUT_GENERAL,
+ output_layers),
+ barrier(history, VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
+ VK_IMAGE_LAYOUT_GENERAL, history_layers),
+ };
+ cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT |
+ VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
+ VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, {}, before);
+ const VkImageCopy copy{
+ .srcSubresource = output_layers,
+ .srcOffset = {0, 0, 0},
+ .dstSubresource = history_layers,
+ .dstOffset = {0, 0, 0},
+ .extent = {extent.width, extent.height, 1},
+ };
+ cmdbuf.CopyImage(output, VK_IMAGE_LAYOUT_GENERAL, history, VK_IMAGE_LAYOUT_GENERAL, copy);
+ const std::array after{
+ barrier(output, VK_ACCESS_TRANSFER_READ_BIT,
+ VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT |
+ VK_ACCESS_TRANSFER_READ_BIT,
+ VK_IMAGE_LAYOUT_GENERAL, output_layers),
+ barrier(history, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT,
+ VK_IMAGE_LAYOUT_GENERAL, history_layers),
+ };
+ cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0,
+ {}, {}, after);
+ });
+ return true;
+}
+
void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst, ImageView& src,
const Region2D& dst_region, const Region2D& src_region,
Tegra::Engines::Fermi2D::Filter filter,
@@ -2026,10 +2218,22 @@ u64 TextureCacheRuntime::GetDeviceMemoryUsage() const {
return device.GetDeviceMemoryUsage();
}
+u64 TextureCacheRuntime::CurrentTick() const noexcept {
+ return scheduler.CurrentTick();
+}
+
+bool TextureCacheRuntime::IsFree(u64 tick) const noexcept {
+ return scheduler.IsFree(tick);
+}
+
u64 TextureCacheRuntime::GetDeviceMemoryBudget() const {
return device.GetDeviceMemoryBudget();
}
+u64 TextureCacheRuntime::GetDeviceMemorySlack() const {
+ return device.GetDeviceMemorySlack();
+}
+
bool TextureCacheRuntime::CanDownloadMsaa(const VideoCommon::ImageInfo& info) const {
const VkImageAspectFlags aspect_mask = ImageAspectMask(info.format);
if ((aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) != 0) {
@@ -2107,10 +2311,33 @@ void TextureCacheRuntime::ReleaseMsaaScratchImage(VkImage image) {
}
void TextureCacheRuntime::TickFrame() {
+ static constexpr u32 MAX_UNUSED_SCRATCH_FRAMES = 60;
+ // Finished decode images wait for the next texture of their shape, up to this much memory.
+ static constexpr u64 MaxFreeTranscodeBytes = u64{256} << 20;
+ const auto decode_bytes = [](const PendingTranscode& t) {
+ u64 bytes{};
+ for (u32 level = 0; level < t.levels; ++level) {
+ bytes += u64{(std::max)(t.width >> level, 1U)} * (std::max)(t.height >> level, 1U) * 4;
+ }
+ return bytes * t.layers;
+ };
+ u64 free_bytes{};
+ for (const PendingTranscode& t : free_transcodes) {
+ free_bytes += decode_bytes(t);
+ }
while (!pending_transcodes.empty() && scheduler.IsFree(pending_transcodes.front().tick)) {
+ PendingTranscode& done = pending_transcodes.front();
+ const u64 bytes = decode_bytes(done);
+ if (free_bytes + bytes <= MaxFreeTranscodeBytes) {
+ done.unused_frames = 0;
+ free_bytes += bytes;
+ free_transcodes.push_back(std::move(done));
+ }
pending_transcodes.pop_front();
}
- static constexpr u32 MAX_UNUSED_SCRATCH_FRAMES = 60;
+ std::erase_if(free_transcodes, [](PendingTranscode& t) {
+ return ++t.unused_frames > MAX_UNUSED_SCRATCH_FRAMES;
+ });
std::erase_if(msaa_scratch_images, [this](MsaaScratchImage& scratch) {
if (!scheduler.IsFree(scratch.tick)) {
scratch.unused_frames = 0;
@@ -2650,7 +2877,24 @@ bool Image::InitSparseLayers() {
const u32 block_width = huge_astc ? 1 : VideoCore::Surface::DefaultBlockWidth(info.format);
const u32 block_height = huge_astc ? 1 : VideoCore::Surface::DefaultBlockHeight(info.format);
const u64 block_bytes = huge_astc ? 4 : VideoCore::Surface::BytesPerBlock(info.format);
- const VkExtent3D granularity = it->formatProperties.imageGranularity;
+ VkExtent3D granularity = it->formatProperties.imageGranularity;
+ if (granularity.width == 0 || granularity.height == 0) {
+ // Seen with AMD's driver: the sparse block size comes back as 0, and the size computation
+ // below divided by it. Use the standard 2D block shape when the device promises it.
+ if (!StandardSparseBlockShape(device, block_bytes, granularity)) {
+ LOG_WARNING(Render_Vulkan,
+ "Sparse layered image {:#x}: the driver reports no sparse block size and "
+ "no standard block shape; not using sparse layers",
+ gpu_addr);
+ return false;
+ }
+ granularity.width *= block_width;
+ granularity.height *= block_height;
+ LOG_INFO(Render_Vulkan,
+ "Sparse layered image {:#x}: no block size from the driver, using the standard "
+ "{}x{} texel block",
+ gpu_addr, granularity.width, granularity.height);
+ }
s.tile_width = granularity.width;
s.tile_height = granularity.height;
const u64 tile = s.requirements.alignment;
@@ -4087,30 +4331,47 @@ void TextureCacheRuntime::TranscodeAstcToBcn(
const u32 levels = static_cast<u32>(info.resources.levels);
const u32 layers = static_cast<u32>(info.resources.layers);
constexpr VkFormat DecodeFormat = VK_FORMAT_A8B8G8R8_UNORM_PACK32;
- PendingTranscode pending{
- .tick = 0,
- .image = memory_allocator.CreateImage(VkImageCreateInfo{
- .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
- .pNext = nullptr,
- .flags = 0,
- .imageType = VK_IMAGE_TYPE_2D,
- .format = DecodeFormat,
- .extent = {width, height, 1},
- .mipLevels = levels,
- .arrayLayers = layers,
- .samples = VK_SAMPLE_COUNT_1_BIT,
- .tiling = VK_IMAGE_TILING_OPTIMAL,
- .usage = VK_IMAGE_USAGE_STORAGE_BIT,
- .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
- .queueFamilyIndexCount = 0,
- .pQueueFamilyIndices = nullptr,
- .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
- }, true),
- .views = {},
- };
- for (u32 level = 0; level < levels; ++level) {
- pending.views.push_back(
- MakeStorageView(device.GetLogical(), level, *pending.image, DecodeFormat));
+ // A decode image of the same shape the GPU is done with: title 1's title screen decodes
+ // dozens of textures a frame, mostly of a few shapes. Making one took 263 us; from VMA's
+ // shared blocks 90 us, but freeing them there left ~1 GB of those blocks unused.
+ const auto reusable = std::ranges::find_if(free_transcodes, [&](const PendingTranscode& t) {
+ return t.width == width && t.height == height && t.levels == levels && t.layers == layers;
+ });
+ PendingTranscode pending;
+ if (reusable != free_transcodes.end()) {
+ pending = std::move(*reusable);
+ free_transcodes.erase(reusable);
+ } else {
+ pending = PendingTranscode{
+ .tick = 0,
+ .image = memory_allocator.CreateImage(VkImageCreateInfo{
+ .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
+ .pNext = nullptr,
+ .flags = 0,
+ .imageType = VK_IMAGE_TYPE_2D,
+ .format = DecodeFormat,
+ .extent = {width, height, 1},
+ .mipLevels = levels,
+ .arrayLayers = layers,
+ .samples = VK_SAMPLE_COUNT_1_BIT,
+ .tiling = VK_IMAGE_TILING_OPTIMAL,
+ .usage = VK_IMAGE_USAGE_STORAGE_BIT,
+ .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
+ .queueFamilyIndexCount = 0,
+ .pQueueFamilyIndices = nullptr,
+ .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
+ }, true),
+ .views = {},
+ .width = width,
+ .height = height,
+ .levels = levels,
+ .layers = layers,
+ .unused_frames = 0,
+ };
+ for (u32 level = 0; level < levels; ++level) {
+ pending.views.push_back(
+ MakeStorageView(device.GetLogical(), level, *pending.image, DecodeFormat));
+ }
}
astc_decoder_pass->AssembleInto(
info, 0, image.guest_size_bytes,
@@ -4119,7 +4380,7 @@ void TextureCacheRuntime::TranscodeAstcToBcn(
.aspect_mask = VK_IMAGE_ASPECT_COLOR_BIT,
.initialized = false,
.level_view = [&pending](s32 level) { return *pending.views[level]; },
- .wait = false,
+ .wait = true,
},
map, swizzles);
diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h
index 43218d6ad3..564b2cbf9e 100644
--- a/src/video_core/renderer_vulkan/vk_texture_cache.h
+++ b/src/video_core/renderer_vulkan/vk_texture_cache.h
@@ -67,10 +67,22 @@ public:
u64 GetDeviceMemoryUsage() const;
+ /// GPU progress: the texture cache frees a deleted image once the GPU is past its last use.
+ [[nodiscard]] u64 CurrentTick() const noexcept;
+
+ [[nodiscard]] bool IsFree(u64 tick) const noexcept;
+
u64 GetDeviceMemoryBudget() const;
+ u64 GetDeviceMemorySlack() const;
+
bool CanReportMemoryUsage() const;
+ /// Images that did not fit the device memory budget so far (see MemoryAllocator).
+ u32 MemoryPressureEvents() const noexcept {
+ return memory_allocator.MemoryPressureEvents();
+ }
+
void LogMemoryStatistics() const;
/// True when ASTC is re-compressed to BC3 and opaque images may use BC1 instead.
@@ -87,6 +99,12 @@ public:
Tegra::Engines::Fermi2D::Filter filter,
Tegra::Engines::Fermi2D::Operation operation);
+ /// Stands in for a DLSS evaluation: temporal upscale of input_color into dst, accumulated with
+ /// a copy of dst's previous contents (the last output). False when dst cannot take it.
+ bool TemporalUpscale(Framebuffer* dst_framebuffer, ImageView& dst, ImageView& input_color,
+ ImageView& motion_vectors, std::array<f32, 2> jitter,
+ std::array<f32, 2> motion_scale);
+
void CopyImage(Image& dst, Image& src, std::span<const VideoCommon::ImageCopy> copies);
void CopyImageMSAA(Image& dst, Image& src, std::span<const VideoCommon::ImageCopy> copies);
@@ -169,6 +187,15 @@ public:
StagingBufferPool& staging_buffer_pool;
BlitImageHelper& blit_image_helper;
RenderPassCache& render_pass_cache;
+
+ /// TemporalUpscale's history: its previous result, copied right after drawing it.
+ struct TemporalHistory {
+ vk::Image image;
+ vk::ImageView view;
+ VkFormat format{};
+ VkExtent2D extent{};
+ } temporal_history;
+
std::optional<ASTCDecoderPass> astc_decoder_pass;
std::optional<BcEncoderPass> bc_encoder_pass;
@@ -177,13 +204,20 @@ public:
void TranscodeAstcToBcn(Image& image, const StagingBufferRef& map,
std::span<const VideoCommon::SwizzleParameters> swizzles);
- /// Temporary decode images, freed once the GPU is past `tick`.
+ /// Temporary decode images, reusable once the GPU is past `tick`.
struct PendingTranscode {
u64 tick;
vk::Image image;
std::vector<vk::ImageView> views;
+ u32 width;
+ u32 height;
+ u32 levels;
+ u32 layers;
+ u32 unused_frames;
};
std::deque<PendingTranscode> pending_transcodes;
+ /// Decode images the GPU is done with, for the next texture of the same shape.
+ std::vector<PendingTranscode> free_transcodes;
std::optional<BlockLinearUnswizzle3DPass> bl3d_unswizzle_pass;
const Settings::ResolutionScalingInfo& resolution;
@@ -375,6 +409,11 @@ public:
return (this->*current_image).UsageFlags();
}
+ /// Where the image's memory lies, for the collector (TextureCache::RunGarbageCollector).
+ [[nodiscard]] vk::Image::MemoryBlock MemoryBlock() const noexcept {
+ return original_image.GetMemoryBlock();
+ }
+
/// Returns true when the image is already initialized and mark it as initialized
[[nodiscard]] bool ExchangeInitialization() noexcept {
return std::exchange(initialized, true);
diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.cpp b/src/video_core/renderer_vulkan/vk_update_descriptor.cpp
index 742a8d9969..29f3ee71a1 100644
--- a/src/video_core/renderer_vulkan/vk_update_descriptor.cpp
+++ b/src/video_core/renderer_vulkan/vk_update_descriptor.cpp
@@ -20,10 +20,10 @@ UpdateDescriptorQueue::UpdateDescriptorQueue(const Device& device_, size_t frame
bool supports_descriptor_buffer_)
: device{device_}, frame_payload_size{frame_payload_size_},
supports_descriptor_buffer{supports_descriptor_buffer_},
- payload(frame_payload_size_ * FRAMES_IN_FLIGHT)
+ payload(std::make_unique_for_overwrite<DescriptorUpdateEntry[]>(frame_payload_size_ * FRAMES_IN_FLIGHT))
{
- payload_start = payload.data();
- payload_cursor = payload.data();
+ payload_start = payload.get();
+ payload_cursor = payload.get();
}
UpdateDescriptorQueue::~UpdateDescriptorQueue() = default;
@@ -32,7 +32,7 @@ void UpdateDescriptorQueue::TickFrame() {
if (++frame_index >= FRAMES_IN_FLIGHT) {
frame_index = 0;
}
- payload_start = payload.data() + frame_index * frame_payload_size;
+ payload_start = payload.get() + frame_index * frame_payload_size;
payload_cursor = payload_start;
}
diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.h b/src/video_core/renderer_vulkan/vk_update_descriptor.h
index 9f30e65dbd..4d393376b4 100644
--- a/src/video_core/renderer_vulkan/vk_update_descriptor.h
+++ b/src/video_core/renderer_vulkan/vk_update_descriptor.h
@@ -6,6 +6,7 @@
#pragma once
+#include <memory>
#include <variant>
#include <vector>
#include "video_core/vulkan_common/vulkan_wrapper.h"
@@ -122,7 +123,7 @@ private:
DescriptorUpdateEntry* payload_cursor = nullptr;
DescriptorUpdateEntry* payload_start = nullptr;
const DescriptorUpdateEntry* upload_start = nullptr;
- std::vector<DescriptorUpdateEntry> payload;
+ std::unique_ptr<DescriptorUpdateEntry[]> payload;
};
// TODO: should these be separate classes instead?