// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: 2023 yuzu Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later #pragma once #include #include #include #include #include #include #include // TODO: find out which don't require stable iters #include #include "common/container/unordered_map.h" #include "common/container/unordered_set.h" #include #include #include #include #include "common/common_types.h" #include "common/spin_recursive_mutex.h" #include "common/hash.h" #include "common/literals.h" #include "common/lru_cache.h" #include #include "common/scratch_buffer.h" #include "common/slot_vector.h" #include "video_core/texture_cache/pending_download.h" #include "common/thread_worker.h" #include "video_core/compatible_formats.h" #include "video_core/control/channel_state_cache.h" #include "video_core/delayed_destruction_ring.h" #include "video_core/engines/fermi_2d.h" #include "video_core/surface.h" #include "video_core/texture_cache/descriptor_table.h" #include "video_core/texture_cache/image_base.h" #include "video_core/texture_cache/image_info.h" #include "video_core/texture_cache/image_view_base.h" #include "video_core/texture_cache/render_targets.h" #include "video_core/texture_cache/types.h" #include "video_core/textures/texture.h" namespace Tegra { namespace Control { struct ChannelState; } } // namespace Tegra namespace VideoCommon { using Tegra::Texture::TICEntry; using Tegra::Texture::TSCEntry; using VideoCore::Surface::PixelFormat; using namespace Common::Literals; struct ImageViewInOut { u32 index{}; bool blacklist{}; ImageViewId id{}; }; struct AsyncDecodeContext { ImageId image_id; Common::ScratchBuffer decoded_data; boost::container::small_vector copies; std::mutex mutex; std::atomic_bool complete; bool cancelled{}; ///< The image was deleted meanwhile: drop the result (GPU thread only). }; using TextureCacheGPUMap = ::Common::unordered_map, Common::IdentityHash>; class TextureCacheChannelInfo : public ChannelInfo { public: TextureCacheChannelInfo() = delete; TextureCacheChannelInfo(Tegra::Control::ChannelState& state) noexcept; TextureCacheChannelInfo(const TextureCacheChannelInfo& state) = delete; TextureCacheChannelInfo& operator=(const TextureCacheChannelInfo&) = delete; DescriptorTable graphics_image_table; DescriptorTable graphics_sampler_table; DescriptorTable compute_image_table; DescriptorTable compute_sampler_table; // TODO: still relies on bad iterators :( std::unordered_map image_views; std::unordered_map samplers; ::Common::unordered_map sampler_ids; ::Common::unordered_map image_view_ids; TextureCacheGPUMap* gpu_page_table = nullptr; TextureCacheGPUMap* sparse_page_table = nullptr; }; template class TextureCache : public VideoCommon::ChannelSetupCaches { /// Address shift for caching images into a hash table static constexpr u64 YUZU_PAGEBITS = 20; /// Enables debugging features to the texture cache static constexpr bool ENABLE_VALIDATION = P::ENABLE_VALIDATION; /// Implement blits as copies between framebuffers static constexpr bool FRAMEBUFFER_BLITS = P::FRAMEBUFFER_BLITS; /// True when some copies have to be emulated static constexpr bool HAS_EMULATED_COPIES = P::HAS_EMULATED_COPIES; /// True when the API can provide info about the memory of the device. static constexpr bool HAS_DEVICE_MEMORY_INFO = P::HAS_DEVICE_MEMORY_INFO; /// True when the API can do asynchronous texture downloads. static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = P::IMPLEMENTS_ASYNC_DOWNLOADS; static constexpr size_t UNSET_CHANNEL{(std::numeric_limits::max)()}; #ifdef YUZU_LEGACY static constexpr s64 TARGET_THRESHOLD = 3_GiB; #else static constexpr s64 TARGET_THRESHOLD = 4_GiB; #endif // Periodic statistics (TickFrame). u64 gc_deletes{}; u64 gc_costly_deletes{}; u64 gc_download_deletes{}; ///< Evictions that downloaded the image first (a full GPU wait) u64 async_decodes_queued{}; u64 images_created{}; u64 images_uploaded{}; ///< Image contents uploaded from guest memory (last 600 frames) u64 uploaded_bytes{}; ///< Guest bytes those uploads read u64 largest_upload{}; ///< Largest of them u64 layer_uploads{}; ///< Of them, uploads of only the written layers bool native_thresholds_applied{}; u64 native_critical_memory{}; u64 native_expected_memory{}; u64 budget_check_frames{}; u32 seen_pressure_events{}; ///< Allocator pressure events already answered u64 pressure_margin{}; ///< Extra room under the budget after images did not fit u64 pressure_quiet_frames{}; ///< Frames since the last such image u64 slack_margin{}; ///< Extra room for allocator block space the driver counts as used u64 astc_opaque_images{}; u64 astc_alpha_images{}; std::vector astc_scan_buffer; u64 images_deleted{}; std::map> created_shapes; static constexpr s64 DEFAULT_EXPECTED_MEMORY = 1_GiB + 125_MiB; static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB + 625_MiB; /// The most device memory the collector lets the cache fill before it collects in earnest, /// on every GPU (ApplyThresholdOverrides). static constexpr u64 MaxCriticalMemory = 5_GiB; static constexpr size_t GC_EMERGENCY_COUNTS = 2; using Runtime = typename P::Runtime; using Image = typename P::Image; using ImageAlloc = typename P::ImageAlloc; using ImageView = typename P::ImageView; using Sampler = typename P::Sampler; using Framebuffer = typename P::Framebuffer; using AsyncBuffer = typename P::AsyncBuffer; using BufferType = typename P::BufferType; static constexpr u32 LayerUploadMinSize = 64U << 20; static constexpr size_t LargeScratchCapacity = 64U << 20; /// Large layered images whose refresh can decode only the layers written since the last one. [[nodiscard]] bool UploadsByLayer(const ImageBase& image) const noexcept; /// Records the layers a write of [cpu_addr, +size) touches. void MarkLayersWritten(Image& image, DAddr cpu_addr, size_t size); /// SparseTiles images: notes a written range, uploaded with the next refresh. void MarkRangeWritten(Image& image, DAddr cpu_addr, size_t size); /// SparseTiles images: binds and uploads the parts written since the last upload. void RefreshSparseTiles(Image& image); /// Image::BindTiles, counting what it binds in total_used_memory. bool BindSparseTiles(Image& image, std::span rects); /// Image::BackTilesWhole, counting what it binds in total_used_memory. bool BackSparseTilesWhole(Image& image); /// Uploads the image's dirty layers. False when it needs a full upload instead. bool UploadWrittenLayers(Image& image); /// Gives the written layers of a SparseLayers image memory and uploads them. void RefreshSparseLayers(Image& image); /// Image::BindLayers, with the memory it binds counted in total_used_memory. std::vector BindSparseLayers(Image& image, std::span layers); /// Uploads `layers` of the image from guest memory. void UploadLayers(Image& image, std::span layers); struct PendingUnswizzle { ImageId image_id; VideoCommon::ImageInfo info; size_t current_offset = 0; size_t total_size = 0; AsyncBuffer staging_buffer; size_t last_submitted_offset = 0; size_t bytes_per_slice; bool initialized = false; }; struct BlitImages { ImageId dst_id; ImageId src_id; PixelFormat dst_format; PixelFormat src_format; }; public: explicit TextureCache(Runtime&, Tegra::MaxwellDeviceMemoryManager&); /// Notify the cache that a new frame has been queued void TickFrame(); /// Return a constant reference to the given image view id [[nodiscard]] const ImageView& GetImageView(ImageViewId id) const noexcept; /// Return a reference to the given image view id [[nodiscard]] ImageView& GetImageView(ImageViewId id) noexcept; /// Get the imageview from the graphics descriptor table in the specified index [[nodiscard]] ImageView& GetImageView(u32 index) noexcept; /// Mark an image as modified from the GPU void MarkModification(ImageId id) noexcept; /// Fill image_view_ids with the graphics/compute images in indices void FillImageViews(std::span views, bool compute, bool blacklist = true); /// Handle feedback loops during draws. void CheckFeedbackLoop(std::span views); /// Get the sampler from the graphics/compute descriptor table in the specified index Sampler* GetSampler(u32 index, bool compute); /// Get the sampler id from the graphics/compute descriptor table in the specified index SamplerId GetSamplerId(u32 index, bool compute); /// Return a constant reference to the given sampler id [[nodiscard]] const Sampler& GetSampler(SamplerId id) const noexcept; /// Return a reference to the given sampler id [[nodiscard]] Sampler& GetSampler(SamplerId id) noexcept; /// Refresh the state for graphics/compute image view and sampler descriptors void SynchronizeDescriptors(bool compute); /// Updates the Render Targets if they can be rescaled /// @retval True if the Render Targets have been rescaled. bool RescaleRenderTargets(); /// Update bound render targets and upload memory if necessary /// @param is_clear True when the render targets are being used for clears void UpdateRenderTargets(bool is_clear); /// Find a framebuffer with the currently bound render targets /// UpdateRenderTargets should be called before this Framebuffer* GetFramebuffer(); /// Mark images in a range as modified from the CPU void WriteMemory(DAddr cpu_addr, size_t size); /// Download contents of host images to guest memory in a region void DownloadMemory(DAddr cpu_addr, size_t size); std::optional GetFlushArea(DAddr cpu_addr, u64 size); /// Remove images in a region void UnmapMemory(DAddr cpu_addr, size_t size); /// Remove images in a region void UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t size); /// Blit an image with the given parameters bool BlitImage(const Tegra::Engines::Fermi2D::Surface& dst, const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Config& copy); /// Stand-in for a DLSS evaluation: temporal upscale of src into dst (see the runtime's /// TemporalUpscale). False when the images cannot take it; the caller then blits. bool TemporalUpscale(const Tegra::Engines::Fermi2D::Surface& dst, const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Surface& motion_vectors, const Tegra::Texture::TICEntry* depth, std::array jitter, std::array motion_scale); /// Try to find a cached image view in the given CPU address [[nodiscard]] std::pair TryFindFramebufferImageView( const Tegra::FramebufferConfig& config, DAddr cpu_addr); /// Return true when there are uncommitted images to be downloaded [[nodiscard]] bool HasUncommittedFlushes() const noexcept; /// Whether CommitAsyncFlushes would commit anything (downloads or their buffers). [[nodiscard]] bool HasAsyncFlushWork() const noexcept; /// Return true when the caller should wait for async downloads [[nodiscard]] bool ShouldWaitAsyncFlushes() const noexcept; /// Commit asynchronous downloads void CommitAsyncFlushes(); /// Pop asynchronous downloads void PopAsyncFlushes(); [[nodiscard]] ImageId DmaImageId(const Tegra::DMA::ImageOperand& operand, bool is_upload); [[nodiscard]] std::pair DmaBufferImageCopy( const Tegra::DMA::ImageCopy& copy_info, const Tegra::DMA::BufferOperand& buffer_operand, const Tegra::DMA::ImageOperand& image_operand, ImageId image_id, bool modifies_image); void DownloadImageIntoBuffer(Image* image, BufferType buffer, size_t buffer_offset, std::span copies, GPUVAddr address = 0, size_t size = 0); /// Return true when a CPU region is modified from the GPU [[nodiscard]] bool IsRegionGpuModified(DAddr addr, size_t size); [[nodiscard]] bool IsRescaling() const noexcept; [[nodiscard]] bool IsRescaling(const ImageViewBase& image_view) const noexcept; /// Create channel state. void CreateChannel(Tegra::Control::ChannelState& channel) final override; /// Prepare an image to be used void PrepareImage(ImageId image_id, bool is_modification, bool invalidate); Common::SpinRecursiveMutex mutex; private: /// Iterate over all page indices in a range template static void ForEachCPUPage(DAddr addr, size_t size, Func&& func) { static constexpr bool RETURNS_BOOL = std::is_same_v, bool>; const u64 page_end = (addr + size - 1) >> YUZU_PAGEBITS; for (u64 page = addr >> YUZU_PAGEBITS; page <= page_end; ++page) { if constexpr (RETURNS_BOOL) { if (func(page)) { break; } } else { func(page); } } } template static void ForEachGPUPage(GPUVAddr addr, size_t size, Func&& func) { static constexpr bool RETURNS_BOOL = std::is_same_v, bool>; const u64 page_end = (addr + size - 1) >> YUZU_PAGEBITS; for (u64 page = addr >> YUZU_PAGEBITS; page <= page_end; ++page) { if constexpr (RETURNS_BOOL) { if (func(page)) { break; } } else { func(page); } } } void OnGPUASRegister(size_t map_id) final override; /// Runs the Garbage Collector. void RunGarbageCollector(); /// Find or create an image view in the guest descriptor table ImageViewId VisitImageView(u32 index, bool compute); /// Find or create a framebuffer with the given render target parameters FramebufferId GetFramebufferId(const RenderTargets& key); /// Refresh the contents (pixel data) of an image [[nodiscard]] bool IsDownloadable(const ImageBase& image) const noexcept; void RefreshContents(Image& image, ImageId image_id); /// Upload data from guest to an image template void UploadImageContents(Image& image, StagingBuffer& staging_buffer); /// Create a new image view from a guest descriptor [[nodiscard]] ImageViewId CreateImageView(const TICEntry& config); /// Find or create an image from the given parameters [[nodiscard]] ImageId FindOrInsertImage(const ImageInfo& info, GPUVAddr gpu_addr, RelaxedOptions options = RelaxedOptions{}); /// Find an image from the given parameters [[nodiscard]] ImageId FindImage(const ImageInfo& info, GPUVAddr gpu_addr, RelaxedOptions options); /// Create an image from the given parameters [[nodiscard]] ImageId InsertImage(const ImageInfo& info, GPUVAddr gpu_addr, RelaxedOptions options); /// Create a new image and join perfectly matching existing images /// Remove joined images from the cache [[nodiscard]] ImageId JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DAddr cpu_addr); [[nodiscard]] ImageId FindDMAImage(const ImageInfo& info, GPUVAddr gpu_addr); /// Return a blit image pair from the given guest blit parameters [[nodiscard]] std::optional GetBlitImages( const Tegra::Engines::Fermi2D::Surface& dst, const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Config& copy); /// Find or create a sampler from a guest descriptor sampler [[nodiscard]] SamplerId FindSampler(const TSCEntry& config, bool compute); /// Find or create an image view for the given color buffer index [[nodiscard]] ImageViewId FindColorBuffer(size_t index); /// Find or create an image view for the depth buffer [[nodiscard]] ImageViewId FindDepthBuffer(); /// Find or create a view for a render target with the given image parameters [[nodiscard]] ImageViewId FindRenderTargetView(const ImageInfo& info, GPUVAddr gpu_addr); /// Iterates over all the images in a region calling func template void ForEachImageInRegion(DAddr cpu_addr, size_t size, Func&& func); template void ForEachImageInRegionGPU(size_t as_id, GPUVAddr gpu_addr, size_t size, Func&& func); template void ForEachSparseImageInRegion(size_t as_id, GPUVAddr gpu_addr, size_t size, Func&& func); /// Iterates over all the images in a region calling func template void ForEachSparseSegment(ImageBase& image, Func&& func); /// Find or create an image view in the given image with the passed parameters [[nodiscard]] ImageViewId FindOrEmplaceImageView(ImageId image_id, const ImageViewInfo& info); /// Register image in the page table void RegisterImage(ImageId image); /// Unregister image from the page table void UnregisterImage(ImageId image); /// Track CPU reads and writes for image void TrackImage(ImageBase& image, ImageId image_id); /// Stop tracking CPU reads and writes for image void UntrackImage(ImageBase& image, ImageId image_id); /// Delete image from the cache void DeleteImage(ImageId image, bool immediate_delete = false); /// Remove image views references from the cache void RemoveImageViewReferences(std::span removed_views); /// Per image view, the (channel, descriptor) entries of the channels' image_views maps that /// point to it. Deleting an image erased them by scanning every map (title 1: ~5,000 entries, /// 50-130 us for each deleted image, dozens a frame while the collector runs). std::unordered_map, 2>> image_view_descriptors; /// Remove framebuffers using the given image views from the cache void RemoveFramebuffers(std::span removed_views); /// Mark an image as modified from the GPU void MarkModification(ImageBase& image) noexcept; /// Synchronize image aliases, copying data if needed void SynchronizeAliases(ImageId image_id); /// Prepare an image view to be used void PrepareImageView(ImageViewId image_view_id, bool is_modification, bool invalidate); /// Execute copies from one image to the other, even if they are incompatible void CopyImage(ImageId dst_id, ImageId src_id, std::vector copies); /// Bind an image view as render target, downloading resources preemtively if needed void BindRenderTarget(ImageViewId* old_id, ImageViewId new_id); /// Create a render target from a given image and image view parameters [[nodiscard]] std::pair RenderTargetFromImage( ImageId, const ImageViewInfo& view_info); /// Returns true if the current clear parameters clear the whole image of a given image view [[nodiscard]] bool IsFullClear(ImageViewId id); [[nodiscard]] std::pair PrepareDmaImage(ImageId dst_id, GPUVAddr base_addr, bool mark_as_modified); bool ImageCanRescale(ImageBase& image); void InvalidateScale(Image& image); bool ScaleUp(Image& image); bool ScaleDown(Image& image); u64 GetScaledImageSizeBytes(const ImageBase& image); /// The device memory an image counts for in total_used_memory between two driver readings: /// none for an unbacked image (too large to back, or over unwritten sparse blocks), what is /// bound to its layers for a SparseLayers one. u64 ImageMemoryBytes(const Image& image) const; void QueueAsyncDecode(Image& image, ImageId image_id); void TickAsyncDecode(); void QueueAsyncUnswizzle(Image& image, ImageId image_id); void TickAsyncUnswizzle(); Runtime& runtime; Tegra::MaxwellDeviceMemoryManager& device_memory; std::deque gpu_page_table_storage; RenderTargets render_targets; u64 render_targets_serial = 0; u32 rt_active_mask = 0; std::array rt_image_id{}; ImageId rt_depth_image_id{}; u64 texture_bindings_serial = 0; u64 last_feedback_loop_serial = 0; u64 last_feedback_texture_serial = 0; bool last_feedback_loop_result = false; FramebufferId last_framebuffer_id{}; u64 last_framebuffer_serial = 0; ::Common::unordered_map framebuffers; ::Common::unordered_map, Common::IdentityHash> page_table; ::Common::unordered_map> sparse_views; DAddr virtual_invalid_space{}; /// Applies EDEN_NX2_TC_EXPECTED_MIB / EDEN_NX2_TC_CRITICAL_MIB and logs the thresholds. void ApplyThresholdOverrides(bool log = true); /// The CPU address an image at gpu_addr is keyed by: its backing, or a fixed stand-in. [[nodiscard]] std::optional ImageCpuAddress(GPUVAddr gpu_addr, size_t size) const; bool has_deleted_images = false; bool is_rescaling = false; u64 total_used_memory = 0; u64 minimum_memory; u64 expected_memory; u64 critical_memory; size_t gpu_unswizzle_maxsize = 0; size_t swizzle_chunk_size = 0; u32 swizzle_slices_per_batch = 0; struct BufferDownload { GPUVAddr address; size_t size; }; using PendingDownload = PendingImageDownload; Common::SlotVector slot_images; Common::SlotVector slot_map_views; Common::SlotVector slot_image_views; Common::SlotVector slot_image_allocs; Common::SlotVector slot_samplers; Common::SlotVector slot_framebuffers; Common::SlotVector slot_buffer_downloads; // TODO: This data structure is not optimal and it should be reworked std::vector uncommitted_downloads; std::deque> committed_downloads; std::vector uncommitted_async_buffers; std::deque> async_buffers; std::deque async_buffers_death_ring; struct LRUItemParams { using ObjectType = ImageId; using TickType = u64; }; Common::LeastRecentlyUsedCache lru_cache; #ifdef YUZU_LEGACY static constexpr size_t TICKS_TO_DESTROY = 6; #else static constexpr size_t TICKS_TO_DESTROY = 8; #endif DelayedDestructionRing sentenced_images; /// Device memory of the images in each slot of sentenced_images (same index): deleted, but /// counted in the driver's usage until the ring destroys them. std::array sentenced_image_bytes{}; size_t sentenced_image_index = 0; /// Deleted images on runtimes that report GPU progress (CurrentTick, IsFree): destroyed, with /// their device memory, as soon as the GPU is past them rather than TICKS_TO_DESTROY frames /// later. While the collector deletes dozens a frame, eight frames of them held 1-1.5 GB of /// VRAM above its threshold (title 1 on an 8 GB card: 6.6-7.0 GB in use against 5 GiB). struct GpuSentencedImage { u64 tick; u64 bytes; Image image; }; std::deque gpu_sentenced_images; u64 gpu_sentenced_bytes{}; /// Bytes of images per device memory block, for the collector (refreshed every 60 frames). std::unordered_map block_image_bytes; u64 block_image_bytes_frame{~0ULL}; DelayedDestructionRing sentenced_image_view; DelayedDestructionRing sentenced_framebuffers; ::Common::unordered_map image_allocs_table; Common::ScratchBuffer swizzle_data_buffer; Common::ScratchBuffer unswizzle_data_buffer; u64 modification_tick = 0; u64 frame_tick = 0; Common::ThreadWorker texture_decode_worker{1, "TextureDecoder", {}, Common::ThreadPlacement::Efficiency}; std::vector> async_decodes; /// Guest memory images were uploaded from since the last cold hint (device address, size). std::vector> uploaded_ranges; std::deque unswizzle_queue; u8 current_unswizzle_frame; // Join caching boost::container::small_vector join_overlap_ids; ::Common::unordered_set join_overlaps_found; boost::container::small_vector join_left_aliased_ids; boost::container::small_vector join_right_aliased_ids; ::Common::unordered_set join_ignore_textures; boost::container::small_vector join_bad_overlap_ids; struct JoinCopy { bool is_alias; ImageId id; }; boost::container::small_vector join_copies_to_do; ::Common::unordered_map join_alias_indices; }; } // namespace VideoCommon