[vulkan] Removal of QCOM sampler limiters + CustomBorderColor and ColorBorderSwizzle adjustments (#4301)

This PR removes the artificial limit added for QCOM drivers based on a sampler budget limit (based on #3280 work), removes the ban on CustomBorderColor/BorderColorSwizzle for also QCOM driver, reduce the amount of combination required to created a sampler with custom border color/ color border swizzle; adds cases on R16 formats not able to swizzle on BGR565 formats; degrades samplers and color combinations when there's no color border swizzle available; fixes the srgb (LUT) values from legacy and now all the color variations are cached in the same key, instead of having 7 different handlers for them. In resume, rather than performance (due to the reduced amount of process on duplicated/ synced code on pipeline/texture cache), it's a graphica accuracy work that will improve image quality at the cost of almost 0 performance hit, rather than certain hitch if the color combination wasn't cached before. This applies to all platform.

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4301
Reviewed-by: Samuel <lizzie@eden-emu.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>

2 files changed, 43 insertions, 42 deletions

diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp
index 8c18e01f97..1e46ca951c 100644
--- a/src/video_core/vulkan_common/vulkan_device.cpp
+++ b/src/video_core/vulkan_common/vulkan_device.cpp
@@ -508,12 +508,6 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR
         LOG_WARNING(Render_Vulkan, "Qualcomm drivers require scaled vertex format emulation.");
         has_broken_descriptor_aliasing = true;
         LOG_WARNING(Render_Vulkan, "Qualcomm drivers have broken descriptor aliasing.");
-        LOG_WARNING(Render_Vulkan, "Qualcomm drivers have broken custom border color.");
-        RemoveExtensionFeature(extensions.custom_border_color, features.custom_border_color,
-                               VK_EXT_CUSTOM_BORDER_COLOR_EXTENSION_NAME);
-        LOG_WARNING(Render_Vulkan, "Qualcomm drivers have broken border color swizzle.");
-        RemoveExtensionFeature(extensions.border_color_swizzle, features.border_color_swizzle,
-                               VK_EXT_BORDER_COLOR_SWIZZLE_EXTENSION_NAME);
         LOG_WARNING(Render_Vulkan, "Qualcomm drivers have broken color write enable.");
         RemoveExtensionFeature(extensions.color_write_enable, features.color_write_enable,
                                VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME);
@@ -617,21 +611,6 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR
         }
     }
 
-    if (is_qualcomm) {
-        const size_t sampler_limit = properties.properties.limits.maxSamplerAllocationCount;
-        if (sampler_limit > 0) {
-            constexpr size_t MIN_SAMPLER_BUDGET = 1024U;
-            const size_t reserved = sampler_limit / 4U;
-            const size_t derived_budget =
-                (std::max)(MIN_SAMPLER_BUDGET, sampler_limit - reserved);
-            sampler_heap_budget = derived_budget;
-            LOG_WARNING(Render_Vulkan,
-                        "Qualcomm driver reports max {} samplers; reserving {} (25%) and "
-                        "allowing Eden to use {} (75%) to avoid heap exhaustion",
-                        sampler_limit, reserved, sampler_heap_budget);
-        }
-    }
-
     if (extensions.sampler_filter_minmax && is_amd) {
         // Disable ext_sampler_filter_minmax on AMD GCN4 and lower as it is broken.
         if (!features.shader_float16_int8.shaderFloat16) {
@@ -1144,6 +1123,11 @@ bool Device::GetSuitability(bool requires_swapchain) {
             VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_5_PROPERTIES_KHR;
         SetNext(next, properties.maintenance5);
     }
+    if (extensions.custom_border_color) {
+        properties.custom_border_color.sType =
+            VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_PROPERTIES_EXT;
+        SetNext(next, properties.custom_border_color);
+    }
 
     // Perform the property fetch.
     physical.GetProperties2(properties2);
@@ -1240,9 +1224,7 @@ void Device::RemoveUnsuitableExtensions() {
     // VK_EXT_border_color_swizzle
     if (extensions.border_color_swizzle) {
         extensions.border_color_swizzle =
-            extensions.custom_border_color &&
-            features.border_color_swizzle.borderColorSwizzle &&
-            features.border_color_swizzle.borderColorSwizzleFromImage;
+            extensions.custom_border_color && features.border_color_swizzle.borderColorSwizzle;
     }
     RemoveExtensionFeatureIfUnsuitable(extensions.border_color_swizzle,
                                        features.border_color_swizzle,
@@ -1488,11 +1470,26 @@ void Device::SetupFamilies(VkSurfaceKHR surface) {
     }
 }
 
-std::optional<size_t> Device::GetSamplerHeapBudget() const {
-    if (sampler_heap_budget == 0) {
-        return std::nullopt;
+bool Device::TryReserveCustomBorderColorSamplers(size_t count) const {
+    const size_t limit = properties.custom_border_color.maxCustomBorderColorSamplers;
+    if (limit == 0) {
+        return true;
+    }
+    size_t used = custom_border_color_samplers_used.load(std::memory_order_relaxed);
+    while (used + count <= limit) {
+        if (custom_border_color_samplers_used.compare_exchange_weak(
+                used, used + count, std::memory_order_relaxed, std::memory_order_relaxed)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+void Device::ReleaseCustomBorderColorSamplers(size_t count) const {
+    if (count == 0) {
+        return;
     }
-    return sampler_heap_budget;
+    custom_border_color_samplers_used.fetch_sub(count, std::memory_order_relaxed);
 }
 
 u64 Device::GetDeviceMemoryUsage() const {
diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h
index a0aabea1ad..a33ff5cb26 100644
--- a/src/video_core/vulkan_common/vulkan_device.h
+++ b/src/video_core/vulkan_common/vulkan_device.h
@@ -6,6 +6,7 @@
 
 #pragma once
 
+#include <atomic>
 #include <optional>
 #include <set>
 #include <span>
@@ -699,20 +700,18 @@ FN_MAX_LIMIT_LIST
         return features.transform_feedback.geometryStreams;
     }
 
-    /// Returns true if the device supports VK_EXT_custom_border_color.
-    bool IsExtCustomBorderColorSupported() const {
-        return extensions.custom_border_color;
+    /// Returns true if custom border colors can be created without a format.
+    bool IsCustomBorderColorUsable() const {
+        return extensions.custom_border_color &&
+               features.custom_border_color.customBorderColors &&
+               features.custom_border_color.customBorderColorWithoutFormat;
     }
 
-    /// Returns true if customBorderColors feature is available.
-    bool IsCustomBorderColorsSupported() const {
-        return features.custom_border_color.customBorderColors;
-    }
+    /// Takes budget for samplers carrying a custom border color, false when exhausted.
+    bool TryReserveCustomBorderColorSamplers(size_t count) const;
 
-    /// Returns true if customBorderColorWithoutFormat feature is available.
-    bool IsCustomBorderColorWithoutFormatSupported() const {
-        return features.custom_border_color.customBorderColorWithoutFormat;
-    }
+    /// Gives back budget taken by TryReserveCustomBorderColorSamplers.
+    void ReleaseCustomBorderColorSamplers(size_t count) const;
 
     /// Returns true if the device supports VK_EXT_color_write_enable.
     bool IsExtColorWriteEnableSupported() const {
@@ -724,6 +723,12 @@ FN_MAX_LIMIT_LIST
         return extensions.border_color_swizzle;
     }
 
+    /// Returns true if samplers must be carried with border color swizzle mapping.
+    bool NeedsBorderColorSwizzleMapping() const {
+        return extensions.border_color_swizzle &&
+               !features.border_color_swizzle.borderColorSwizzleFromImage;
+    }
+
     /// Returns true if borderColorSwizzleFromImage is available.
     bool IsBorderColorSwizzleFromImageSupported() const {
         return features.border_color_swizzle.borderColorSwizzleFromImage;
@@ -919,8 +924,6 @@ FN_MAX_LIMIT_LIST
         return has_broken_parallel_compiling;
     }
 
-    std::optional<size_t> GetSamplerHeapBudget() const;
-
     /// Returns the vendor name reported from Vulkan.
     std::string_view GetVendorName() const {
         return properties.driver.driverName;
@@ -1173,6 +1176,7 @@ private:
         VkPhysicalDeviceTransformFeedbackPropertiesEXT transform_feedback{};
         VkPhysicalDeviceMaintenance5PropertiesKHR maintenance5{};
         VkPhysicalDeviceDepthStencilResolveProperties depth_stencil_resolve{};
+        VkPhysicalDeviceCustomBorderColorPropertiesEXT custom_border_color{};
 
         VkPhysicalDeviceProperties properties{};
     };
@@ -1211,7 +1215,7 @@ private:
     bool dynamic_state3_alpha_to_coverage{};
     bool dynamic_state3_alpha_to_one{};
     bool supports_conditional_barriers{};      ///< Allows barriers in conditional control flow.
-    size_t sampler_heap_budget{};              ///< Sampler budget for buggy drivers (0 = unlimited).
+    mutable std::atomic<size_t> custom_border_color_samplers_used{};
     u64 device_access_memory{};                ///< Total size of device local memory in bytes.
     u32 sets_per_pool{};                       ///< Sets per Description Pool
     NvidiaArchitecture nvidia_arch{NvidiaArchitecture::Arch_AmpereOrNewer};