diff options
Diffstat (limited to 'src/video_core/texture_cache')
| -rw-r--r-- | src/video_core/texture_cache/descriptor_table.h | 10 | ||||
| -rw-r--r-- | src/video_core/texture_cache/image_base.h | 7 | ||||
| -rw-r--r-- | src/video_core/texture_cache/texture_cache.h | 378 | ||||
| -rw-r--r-- | src/video_core/texture_cache/texture_cache_base.h | 53 | ||||
| -rw-r--r-- | src/video_core/texture_cache/types.h | 9 | ||||
| -rw-r--r-- | src/video_core/texture_cache/util.cpp | 204 | ||||
| -rw-r--r-- | src/video_core/texture_cache/util.h | 21 |
7 files changed, 663 insertions, 19 deletions
diff --git a/src/video_core/texture_cache/descriptor_table.h b/src/video_core/texture_cache/descriptor_table.h index e40c128ab5..75383809c5 100644 --- a/src/video_core/texture_cache/descriptor_table.h +++ b/src/video_core/texture_cache/descriptor_table.h @@ -30,11 +30,17 @@ public: } void Invalidate() noexcept { - std::ranges::fill(read_descriptors, 0); + // Cleared on the next read: the texture cache invalidates every channel's tables for + // each image it deletes, dozens of times a frame while its collector runs. + invalidated = true; } [[nodiscard]] std::pair<T, bool> Read(Tegra::MemoryManager const& gpu_memory, u32 index) noexcept { DEBUG_ASSERT(index <= current_limit); + if (invalidated) [[unlikely]] { + std::ranges::fill(read_descriptors, 0); + invalidated = false; + } const GPUVAddr gpu_addr = current_gpu_addr + index * sizeof(T); std::pair<T, bool> result; gpu_memory.ReadBlockUnsafe(gpu_addr, std::addressof(result.first), sizeof(T)); @@ -60,6 +66,7 @@ public: read_descriptors.resize(Common::DivCeil(num_descriptors, 64U)); old_size = (std::min)(old_size, read_descriptors.size()); std::fill(read_descriptors.begin(), read_descriptors.begin() + old_size, 0); + invalidated = false; // descriptors.resize(num_descriptors); } @@ -68,6 +75,7 @@ public: std::vector<T> descriptors; GPUVAddr current_gpu_addr{}; u32 current_limit{}; + bool invalidated{}; ///< read_descriptors is to be cleared before the next read }; } // namespace VideoCommon diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h index 04254fb417..878c229953 100644 --- a/src/video_core/texture_cache/image_base.h +++ b/src/video_core/texture_cache/image_base.h @@ -56,6 +56,10 @@ enum class ImageFlagBits : u32 { /// A large array over Switch 2 sparse blocks: unbacked, each layer gets device memory when /// something is written to it (Image::BindLayers) and only those layers are uploaded. SparseLayers = 1 << 22, + /// A large single image over Switch 2 sparse blocks: unbacked, each host sparse block gets + /// device memory when the guest memory under it can hold data (Image::BindTiles), and only + /// the parts written are uploaded. + SparseTiles = 1 << 23, }; DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits) @@ -108,6 +112,9 @@ struct ImageBase { /// Large layered images uploaded by layer: one bit per layer written since the last upload /// while CpuModified. Empty while CpuModified means the whole image. std::vector<u64> dirty_layers; + /// SparseTiles images: the guest byte ranges (offset, size) written since the last upload while + /// CpuModified. Empty while CpuModified means all of it that can hold data. + std::vector<std::pair<u64, u64>> dirty_ranges; u64 scale_tick = 0; bool has_scaled = false; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 3b783fc6c6..3f3ae9cafd 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -125,6 +125,7 @@ template <class P> void TextureCache<P>::RunGarbageCollector() { bool high_priority_mode = false; bool aggressive_mode = false; + bool evict_old_costly = false; u64 ticks_to_destroy = 0; size_t num_iterations = 0; const auto Configure = [&](bool allow_aggressive) { @@ -139,7 +140,8 @@ void TextureCache<P>::RunGarbageCollector() { ticks_to_destroy = (std::max)(ticks_to_destroy, min_age); num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10); }; - const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode](ImageId image_id) { + const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode, + &evict_old_costly](ImageId image_id) { if (num_iterations == 0) { return true; } @@ -149,7 +151,8 @@ void TextureCache<P>::RunGarbageCollector() { return false; } const bool must_download = IsDownloadable(image) && False(image.flags & ImageFlagBits::BadOverlap); - if ((!aggressive_mode && True(image.flags & ImageFlagBits::CostlyLoad)) || (!high_priority_mode && must_download)) { + if ((!aggressive_mode && !evict_old_costly && True(image.flags & ImageFlagBits::CostlyLoad)) || + (!high_priority_mode && must_download)) { return false; } if (must_download) { @@ -175,16 +178,84 @@ void TextureCache<P>::RunGarbageCollector() { } return false; }; + // Deleting by age alone left every device memory block partly used: the collector deletes + // thousands of images a minute and new ones land in the holes, so blocks seldom empty (title + // 1: ~0.8 GB of holes in ~190 32 MiB blocks after ten minutes, which the driver counts as + // used). Among the oldest candidates (8 per image this pass may delete), those in the + // emptiest blocks go first, so blocks drain and go back to the driver; placement already + // prefers the fullest blocks (best fit). Images with memory of their own sit in the middle. + const auto Collect = [&] { + if constexpr (requires(const Image& image) { image.MemoryBlock(); }) { + std::vector<ImageId> candidates; + const size_t limit = num_iterations * 8; + lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, [&](ImageId image_id) { + candidates.push_back(image_id); + return candidates.size() >= limit; + }); + if (candidates.empty()) { + return; + } + if (block_image_bytes_frame == ~0ULL || frame_tick - block_image_bytes_frame >= 60) { + block_image_bytes_frame = frame_tick; + block_image_bytes.clear(); + for (const auto [image_id, image] : slot_images) { + const auto block = image->MemoryBlock(); + if (block.block != 0) { + block_image_bytes[block.block] += block.size; + } + } + } + std::vector<std::pair<double, ImageId>> ordered; + ordered.reserve(candidates.size()); + for (const ImageId image_id : candidates) { + const auto block = slot_images[image_id].MemoryBlock(); + double fill = 0.5; + if (block.block != 0 && block.block_size != 0) { + const auto it = block_image_bytes.find(block.block); + fill = it != block_image_bytes.end() + ? static_cast<double>(it->second) / static_cast<double>(block.block_size) + : 0.0; + } + ordered.emplace_back(fill, image_id); + } + std::stable_sort(ordered.begin(), ordered.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); + for (const auto& [fill, image_id] : ordered) { + if (Cleanup(image_id)) { + break; + } + } + } else { + lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + } + }; Configure(false); - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + static const u64 old_age = [] { + const char* value = std::getenv("EDEN_NX2_TC_OLD_AGE"); + return value != nullptr ? std::strtoull(value, nullptr, 10) : 1800ULL; + }(); + if (high_priority_mode && old_age != 0 && frame_tick > old_age) { + evict_old_costly = true; + lru_cache.ForEachItemBelow(frame_tick - old_age, Cleanup); + evict_old_costly = false; + } + Collect(); if (total_used_memory >= critical_memory) { Configure(true); - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); + Collect(); } } template <class P> void TextureCache<P>::ApplyThresholdOverrides(bool log) { + // On every GPU: past what a frame needs, more device memory only keeps the textures of earlier + // scenes (title 1 filled 6.7 GB of an 8 GB card and draws its races in ~5 GB), so collect from + // MaxCriticalMemory on rather than near the card's budget. + critical_memory = (std::min)(critical_memory, MaxCriticalMemory); + expected_memory = + (std::min)(expected_memory, critical_memory - (std::min)(critical_memory, u64{768_MiB})); + minimum_memory = + (std::min)(minimum_memory, expected_memory - (std::min)(expected_memory, u64{1_GiB})); // EDEN_NX2_TC_EXPECTED_MIB / EDEN_NX2_TC_CRITICAL_MIB override the collector's thresholds. for (const auto& [name, target] : {std::pair{"EDEN_NX2_TC_EXPECTED_MIB", &expected_memory}, std::pair{"EDEN_NX2_TC_CRITICAL_MIB", &critical_memory}}) { @@ -230,26 +301,78 @@ void TextureCache<P>::TickFrame() { // budget as the card fills (16 GB card: 13.5 GB left mid-race), hence the re-read every 30 // frames. EDEN_NX2_TC_*_MIB overrides still win. if constexpr (requires { runtime.GetDeviceMemoryBudget(); }) { + if constexpr (requires { runtime.MemoryPressureEvents(); }) { + // Images that did not fit (made in host memory or with nothing bound) mean the + // thresholds sit too high for this card and driver: leave more room until the + // pressure has been gone for a while. + if (native_thresholds_applied) { + const u32 events = runtime.MemoryPressureEvents(); + if (events != seen_pressure_events) { + seen_pressure_events = events; + pressure_margin = (std::min)(pressure_margin + 512_MiB, 3_GiB); + pressure_quiet_frames = 0; + budget_check_frames = 0; // thresholds are recomputed below + LOG_WARNING(HW_GPU, "Out of device memory: keeping {} MiB more free", + pressure_margin >> 20); + } else if (pressure_margin != 0 && ++pressure_quiet_frames % 600 == 0) { + pressure_margin -= (std::min)(pressure_margin, 128_MiB); + budget_check_frames = 0; + } + } + } if (native_thresholds_applied && budget_check_frames++ % 30 == 0) { const u64 budget = runtime.GetDeviceMemoryBudget(); if (budget > 2_GiB) { const u64 old_critical = critical_memory; const u64 margin = (std::min)(budget / 8, 1_GiB); - critical_memory = budget - margin; - expected_memory = budget - 2 * margin; + if constexpr (requires { runtime.GetDeviceMemorySlack(); }) { + // The usage counted here leaves out the unused part of the allocator's + // blocks, which the driver counts and a full heap cannot hand to a new + // image (RX 7800 XT: 1.06 GiB of it, the whole margin, with the heap at + // its budget for a minute before images stopped fitting). Reserve it + // too: at once when it grows, slowly when it shrinks, so evicting images + // (which leaves partly used blocks) does not chase the threshold down. + const u64 slack = (std::min)(runtime.GetDeviceMemorySlack(), 1536_MiB); + slack_margin = slack >= slack_margin + ? slack + : slack_margin - (std::min)(slack_margin - slack, 16_MiB); + } + critical_memory = + budget - (std::min)(margin + pressure_margin + slack_margin, budget / 2); + expected_memory = critical_memory - (std::min)(margin, critical_memory / 2); // Log the first values and real changes (the budget moves by a few MiB all // the time). const u64 change = critical_memory > old_critical ? critical_memory - old_critical : old_critical - critical_memory; - ApplyThresholdOverrides(budget_check_frames == 1 || change >= 256_MiB); + const bool log = budget_check_frames == 1 || change >= 256_MiB; + ApplyThresholdOverrides(log); + if (log) { + LOG_INFO(HW_GPU, + "Memory margins: base {} MiB, out of memory {} MiB, block slack {} MiB", + margin >> 20, pressure_margin >> 20, slack_margin >> 20); + } } } } } + if constexpr (requires { runtime.CurrentTick(); }) { + while (!gpu_sentenced_images.empty() && + runtime.IsFree(gpu_sentenced_images.front().tick)) { + gpu_sentenced_bytes -= gpu_sentenced_images.front().bytes; + gpu_sentenced_images.pop_front(); + } + } // If we can obtain the memory info, use it instead of the estimate. if (runtime.CanReportMemoryUsage()) { total_used_memory = runtime.GetDeviceMemoryUsage(); + // Images the collector deleted keep their memory for TICKS_TO_DESTROY frames: counting + // it kept the collector in aggressive mode, deleting 40+ images a frame for frames after + // it had freed enough (title 1: frames over 100 ms for seconds at a time). + for (const u64 bytes : sentenced_image_bytes) { + total_used_memory -= (std::min)(total_used_memory, bytes); + } + total_used_memory -= (std::min)(total_used_memory, gpu_sentenced_bytes); } // Every 600 frames: device memory in use against the collector's thresholds, and what the // images themselves account for (VRAM pressure makes the collector thrash). @@ -316,10 +439,18 @@ void TextureCache<P>::TickFrame() { RunGarbageCollector(); } sentenced_images.Tick(); + sentenced_image_index = (sentenced_image_index + 1) % TICKS_TO_DESTROY; + sentenced_image_bytes[sentenced_image_index] = 0; sentenced_framebuffers.Tick(); sentenced_image_view.Tick(); TickAsyncDecode(); TickAsyncUnswizzle(); + if (frame_tick % 600 == 0 && !uploaded_ranges.empty()) { + for (const auto& [address, size] : uploaded_ranges) { + device_memory.HintCold(address, size); + } + uploaded_ranges.clear(); + } runtime.TickFrame(); ++frame_tick; @@ -678,6 +809,7 @@ typename P::Framebuffer* TextureCache<P>::GetFramebuffer() { template <class P> ImageViewId TextureCache<P>::VisitImageView(u32 index, bool compute) { auto& table = compute ? channel_state->compute_image_table : channel_state->graphics_image_table; + index = Tegra::Texture::TextureHeaderIndex(index, table.current_limit); if (index > table.current_limit) { LOG_DEBUG(HW_GPU, "Invalid image view index={}", index); return NULL_IMAGE_VIEW_ID; @@ -689,8 +821,13 @@ ImageViewId TextureCache<P>::VisitImageView(u32 index, bool compute) { if (IsValidEntry(*gpu_memory, descriptor)) { // Is new (registered view) on the texture cache side? const auto [pair, is_new_tc] = channel_state->image_views.try_emplace(descriptor); - if (is_new_tc) + if (is_new_tc) { pair->second = CreateImageView(descriptor); + if (pair->second != NULL_IMAGE_VIEW_ID) { + image_view_descriptors[pair->second].emplace_back(current_channel_id, + descriptor); + } + } PrepareImageView(pair->second, false, false); channel_state->image_view_ids.insert_or_assign(map_index, pair->second); return pair->second; @@ -729,6 +866,10 @@ void TextureCache<P>::WriteMemory(DAddr cpu_addr, size_t size) { image.flags |= ImageFlagBits::CpuModified | ImageFlagBits::Remapped; return; } + if (True(image.flags & ImageFlagBits::SparseTiles)) { + MarkRangeWritten(image, cpu_addr, size); + return; + } if (UploadsByLayer(image)) { MarkLayersWritten(image, cpu_addr, size); return; @@ -953,6 +1094,73 @@ bool TextureCache<P>::BlitImage(const Tegra::Engines::Fermi2D::Surface& dst, } template <class P> +bool TextureCache<P>::TemporalUpscale(const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& src, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, + std::array<f32, 2> jitter, std::array<f32, 2> motion_scale) { + if constexpr (FRAMEBUFFER_BLITS) { + return false; + } else { + const Tegra::Engines::Fermi2D::Config copy{ + .operation = Tegra::Engines::Fermi2D::Operation::SrcCopy, + .filter = Tegra::Engines::Fermi2D::Filter::Bilinear, + .must_accelerate = true, + .dst_x0 = 0, + .dst_y0 = 0, + .dst_x1 = static_cast<s32>(dst.width), + .dst_y1 = static_cast<s32>(dst.height), + .src_x0 = 0, + .src_y0 = 0, + .src_x1 = static_cast<s32>(src.width), + .src_y1 = static_cast<s32>(src.height), + }; + const auto result = GetBlitImages(dst, src, copy); + if (!result) { + return false; + } + const ImageInfo motion_info(motion_vectors); + has_deleted_images = false; + const ImageId motion_id = + FindOrInsertImage(motion_info, motion_vectors.Address(), RelaxedOptions{}); + if (has_deleted_images || !motion_id) { + return false; // the colour and output ids may be stale; the caller blits instead + } + const ImageId dst_id = result->dst_id; + const ImageId src_id = result->src_id; + PrepareImage(src_id, false, false); + PrepareImage(motion_id, false, false); + PrepareImage(dst_id, true, false); + for (const ImageId id : {src_id, motion_id, dst_id}) { + if (True(slot_images[id].flags & ImageFlagBits::Rescaled) || + slot_images[id].info.num_samples != 1) { + return false; + } + } + const auto view = [&](ImageId id, const Tegra::Engines::Fermi2D::Surface& surface, + PixelFormat format) -> std::optional<std::pair<FramebufferId, ImageViewId>> { + const std::optional base = slot_images[id].TryFindBase(surface.Address()); + if (!base) { + return std::nullopt; + } + const SubresourceRange range{.base = *base, .extent = {1, 1}}; + return RenderTargetFromImage(id, ImageViewInfo(ImageViewType::e2D, format, range)); + }; + const auto src_view = view(src_id, src, result->src_format); + const auto motion_view = view(motion_id, motion_vectors, motion_info.format); + const auto dst_view = view(dst_id, dst, result->dst_format); + if (!src_view || !motion_view || !dst_view) { + return false; + } + // After the views: creating them may reallocate the framebuffer slots. + Framebuffer* const dst_framebuffer = &slot_framebuffers[dst_view->first]; + return runtime.TemporalUpscale(dst_framebuffer, slot_image_views[dst_view->second], + slot_image_views[src_view->second], + slot_image_views[motion_view->second], jitter, + motion_scale); + } +} + +template <class P> std::pair<typename P::ImageView*, bool> TextureCache<P>::TryFindFramebufferImageView( const Tegra::FramebufferConfig& config, DAddr cpu_addr) { // TODO: Properly implement this @@ -1316,6 +1524,10 @@ void TextureCache<P>::RefreshContents(Image& image, ImageId image_id) { RefreshSparseLayers(image); return; } + if (True(image.flags & ImageFlagBits::SparseTiles)) { + RefreshSparseTiles(image); + return; + } if (!image.dirty_layers.empty() && UploadWrittenLayers(image)) { runtime.InsertUploadMemoryBarrier(); return; @@ -1357,6 +1569,21 @@ void TextureCache<P>::RefreshContents(Image& image, ImageId image_id) { auto staging = runtime.UploadStagingBuffer(MapSizeBytes(image)); UploadImageContents(image, staging); runtime.InsertUploadMemoryBarrier(); + // The host GPU has its own copy now and the guest rarely reads this memory again: TickFrame + // marks it cold, so the OS pages it out before anything else when it needs RAM (title 1 + // streams several GB of textures through it). + constexpr u64 ColdHintMinBytes = 64_KiB; + if (image.guest_size_bytes >= ColdHintMinBytes && + False(image.flags & ImageFlagBits::GpuModified)) { + if (True(image.flags & ImageFlagBits::Sparse)) { + // Non-contiguous: each mapped piece (title 1 maps its own memory into these). + ForEachSparseSegment(image, [&](GPUVAddr, DAddr address, size_t size) { + uploaded_ranges.emplace_back(address, size); + }); + } else { + uploaded_ranges.emplace_back(image.cpu_addr, image.guest_size_bytes); + } + } } template <class P> @@ -1401,6 +1628,77 @@ void TextureCache<P>::MarkLayersWritten(Image& image, DAddr cpu_addr, size_t siz } template <class P> +void TextureCache<P>::MarkRangeWritten(Image& image, DAddr cpu_addr, size_t size) { + if (True(image.flags & ImageFlagBits::CpuModified) && image.dirty_ranges.empty()) { + return; // all of it is dirty already + } + const DAddr begin = (std::max<DAddr>)(cpu_addr, image.cpu_addr); + const DAddr end = (std::min<DAddr>)(cpu_addr + size, image.cpu_addr + image.guest_size_bytes); + if (end <= begin) { + return; + } + image.flags |= ImageFlagBits::CpuModified; + image.dirty_ranges.emplace_back(begin - image.cpu_addr, end - begin); +} + +template <class P> +void TextureCache<P>::RefreshSparseTiles(Image& image) { + if constexpr (requires { &Image::BindTiles; }) { + // The ranges written since the last upload, or, for a new image, all of it that can hold + // data (the tiles the game maps there, or the blocks written). + std::vector<std::pair<u64, u64>> ranges = std::move(image.dirty_ranges); + image.dirty_ranges.clear(); + if (ranges.empty()) { + ranges = Tegra::Nx2SparseDataRanges(image.gpu_addr, image.guest_size_bytes); + } + const std::vector<LevelRect> rects = + SparseTileRects(image.info, ranges, image.SparseTileWidth(), image.SparseTileHeight(), + image.SparseTailLevel()); + if (rects.empty() || !BindSparseTiles(image, rects)) { + runtime.TransitionImageLayout(image); + return; + } + const size_t bytes = UnswizzledRectsBytes(image.info, rects); + auto staging = runtime.UploadStagingBuffer(bytes); + const std::vector<BufferImageCopy> copies = + UnswizzleImageRects(*gpu_memory, image.gpu_addr, image.info, rects, staging.mapped_span); + image.UploadMemory(staging, copies); + runtime.InsertUploadMemoryBarrier(); + ++images_uploaded; + uploaded_bytes += bytes; + } +} + +template <class P> +bool TextureCache<P>::BindSparseTiles(Image& image, std::span<const LevelRect> rects) { + if constexpr (requires { &Image::BindTiles; }) { + const u64 bound_bytes = ImageMemoryBytes(image); + const bool bound = image.BindTiles(rects); + // RegisterImage counts what an image has bound by then; later binds are added here. + if (True(image.flags & ImageFlagBits::Registered)) { + total_used_memory += ImageMemoryBytes(image) - bound_bytes; + } + return bound; + } else { + return false; + } +} + +template <class P> +bool TextureCache<P>::BackSparseTilesWhole(Image& image) { + if constexpr (requires { &Image::BackTilesWhole; }) { + const u64 bound_bytes = ImageMemoryBytes(image); + const bool backed = image.BackTilesWhole(); + if (True(image.flags & ImageFlagBits::Registered)) { + total_used_memory += ImageMemoryBytes(image) - bound_bytes; + } + return backed; + } else { + return false; + } +} + +template <class P> bool TextureCache<P>::UploadWrittenLayers(Image& image) { if constexpr (requires { &Runtime::AccelerateImageUploadLayers; }) { const u32 num_layers = static_cast<u32>(image.info.resources.layers); @@ -1712,9 +2010,14 @@ void TextureCache<P>::InvalidateScale(Image& image) { for (size_t c : active_channel_ids) { auto& channel_info = channel_storage[c]; +#ifndef NDEBUG + // A walk over every entry (title 1: ~5,400 in 5 channels) for each deleted image: debug + // builds only. The tables are invalidated below, so each entry is written again before + // it is read. if constexpr (ENABLE_VALIDATION) for (auto& e : channel_info.image_view_ids) e.second = CORRUPT_ID; +#endif channel_info.graphics_image_table.Invalidate(); channel_info.compute_image_table.Invalidate(); } @@ -2832,6 +3135,8 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) { } // TickFrame sets the total from the driver, which can be under the sum of these estimates. total_used_memory -= (std::min)(total_used_memory, ImageMemoryBytes(slot_images[image_id])); + const u64 freed_bytes = (image.HasScaled() ? GetScaledImageSizeBytes(image) : 0) + + ImageMemoryBytes(slot_images[image_id]); const GPUVAddr gpu_addr = image.gpu_addr; const auto alloc_it = image_allocs_table.find(gpu_addr); if (alloc_it == image_allocs_table.end()) { @@ -2896,19 +3201,32 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) { if (retained) { LOG_INFO(Render_Vulkan, "Retained image for queued download: id={}", image_id.Value()); } else if (!immediate_delete) { - sentenced_images.Push(std::move(slot_images[image_id])); + if constexpr (requires { runtime.CurrentTick(); }) { + gpu_sentenced_images.push_back( + {runtime.CurrentTick(), freed_bytes, std::move(slot_images[image_id])}); + gpu_sentenced_bytes += freed_bytes; + } else { + sentenced_images.Push(std::move(slot_images[image_id])); + sentenced_image_bytes[sentenced_image_index] += freed_bytes; + } } slot_images.erase(image_id); alloc_images.erase(alloc_image_it); if (alloc_images.empty()) { image_allocs_table.erase(alloc_it); + slot_image_allocs.erase(alloc_id); } for (size_t c : active_channel_ids) { auto& channel_info = channel_storage[c]; +#ifndef NDEBUG + // A walk over every entry (title 1: ~5,400 in 5 channels) for each deleted image: debug + // builds only. The tables are invalidated below, so each entry is written again before + // it is read. if constexpr (ENABLE_VALIDATION) for (auto& e : channel_info.image_view_ids) e.second = CORRUPT_ID; +#endif channel_info.graphics_image_table.Invalidate(); channel_info.compute_image_table.Invalidate(); } @@ -2917,17 +3235,23 @@ void TextureCache<P>::DeleteImage(ImageId image_id, bool immediate_delete) { template <class P> void TextureCache<P>::RemoveImageViewReferences(std::span<const ImageViewId> removed_views) { - for (size_t c : active_channel_ids) { - auto& channel_info = channel_storage[c]; - auto it = channel_info.image_views.begin(); - while (it != channel_info.image_views.end()) { - const auto found = std::ranges::find(removed_views, it->second); - if (found != removed_views.end()) { - it = channel_info.image_views.erase(it); - } else { - ++it; + for (const ImageViewId removed : removed_views) { + const auto refs = image_view_descriptors.find(removed); + if (refs == image_view_descriptors.end()) { + continue; + } + for (const auto& [channel, descriptor] : refs->second) { + // The channel may be gone, or its entry point to another view since. + if (std::ranges::find(active_channel_ids, channel) == active_channel_ids.end()) { + continue; + } + auto& image_views = channel_storage[channel].image_views; + if (const auto it = image_views.find(descriptor); + it != image_views.end() && it->second == removed) { + image_views.erase(it); } } + image_view_descriptors.erase(refs); } } @@ -2944,6 +3268,7 @@ void TextureCache<P>::RemoveFramebuffers(std::span<const ImageViewId> removed_vi last_framebuffer_serial = 0; } sentenced_framebuffers.Push(std::move(slot_framebuffers[framebuffer_id])); + slot_framebuffers.erase(framebuffer_id); it = framebuffers.erase(it); } else { ++it; @@ -3025,6 +3350,23 @@ void TextureCache<P>::PrepareImage(ImageId image_id, bool is_modification, bool } } if (is_modification) { + if constexpr (requires { &Image::BackTilesWhole; }) { + if (True(image.flags & ImageFlagBits::SparseTiles) && image.SparseTilesShareZeroBlock()) { + // The render would write the zeroed block all empty blocks share: each gets memory + // of its own first, and the parts that hold data are loaded again. + LOG_INFO(HW_GPU, "GPU write into sparse tiled image {:#x}; backing it whole", + image.gpu_addr); + if (BackSparseTilesWhole(image)) { + image.dirty_ranges.clear(); + RefreshSparseTiles(image); + } else { + LOG_ERROR(HW_GPU, + "Sparse tiled image {:#x}: out of memory backing it whole, the GPU " + "write lands in the shared zeroed block", + image.gpu_addr); + } + } + } if constexpr (requires { &Image::BindLayers; }) { if (True(image.flags & ImageFlagBits::SparseLayers)) { std::vector<u32> layers(static_cast<size_t>(image.info.resources.layers)); diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 94102cd261..09deb8da02 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -136,6 +136,10 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI u64 native_critical_memory{}; u64 native_expected_memory{}; u64 budget_check_frames{}; + u32 seen_pressure_events{}; ///< Allocator pressure events already answered + u64 pressure_margin{}; ///< Extra room under the budget after images did not fit + u64 pressure_quiet_frames{}; ///< Frames since the last such image + u64 slack_margin{}; ///< Extra room for allocator block space the driver counts as used u64 astc_opaque_images{}; u64 astc_alpha_images{}; std::vector<u8> astc_scan_buffer; @@ -144,6 +148,9 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI static constexpr s64 DEFAULT_EXPECTED_MEMORY = 1_GiB + 125_MiB; static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB + 625_MiB; + /// The most device memory the collector lets the cache fill before it collects in earnest, + /// on every GPU (ApplyThresholdOverrides). + static constexpr u64 MaxCriticalMemory = 5_GiB; static constexpr size_t GC_EMERGENCY_COUNTS = 2; using Runtime = typename P::Runtime; @@ -164,6 +171,18 @@ class TextureCache : public VideoCommon::ChannelSetupCaches<TextureCacheChannelI /// Records the layers a write of [cpu_addr, +size) touches. void MarkLayersWritten(Image& image, DAddr cpu_addr, size_t size); + /// SparseTiles images: notes a written range, uploaded with the next refresh. + void MarkRangeWritten(Image& image, DAddr cpu_addr, size_t size); + + /// SparseTiles images: binds and uploads the parts written since the last upload. + void RefreshSparseTiles(Image& image); + + /// Image::BindTiles, counting what it binds in total_used_memory. + bool BindSparseTiles(Image& image, std::span<const LevelRect> rects); + + /// Image::BackTilesWhole, counting what it binds in total_used_memory. + bool BackSparseTilesWhole(Image& image); + /// Uploads the image's dirty layers. False when it needs a full upload instead. bool UploadWrittenLayers(Image& image); @@ -264,6 +283,13 @@ public: const Tegra::Engines::Fermi2D::Surface& src, const Tegra::Engines::Fermi2D::Config& copy); + /// Stand-in for a DLSS evaluation: temporal upscale of src into dst (see the runtime's + /// TemporalUpscale). False when the images cannot take it; the caller then blits. + bool TemporalUpscale(const Tegra::Engines::Fermi2D::Surface& dst, + const Tegra::Engines::Fermi2D::Surface& src, + const Tegra::Engines::Fermi2D::Surface& motion_vectors, + std::array<f32, 2> jitter, std::array<f32, 2> motion_scale); + /// Try to find a cached image view in the given CPU address [[nodiscard]] std::pair<ImageView*, bool> TryFindFramebufferImageView( const Tegra::FramebufferConfig& config, DAddr cpu_addr); @@ -433,6 +459,12 @@ private: /// Remove image views references from the cache void RemoveImageViewReferences(std::span<const ImageViewId> removed_views); + /// Per image view, the (channel, descriptor) entries of the channels' image_views maps that + /// point to it. Deleting an image erased them by scanning every map (title 1: ~5,000 entries, + /// 50-130 us for each deleted image, dozens a frame while the collector runs). + std::unordered_map<ImageViewId, boost::container::small_vector<std::pair<size_t, TICEntry>, 2>> + image_view_descriptors; + /// Remove framebuffers using the given image views from the cache void RemoveFramebuffers(std::span<const ImageViewId> removed_views); @@ -552,6 +584,25 @@ private: static constexpr size_t TICKS_TO_DESTROY = 8; #endif DelayedDestructionRing<Image, TICKS_TO_DESTROY> sentenced_images; + /// Device memory of the images in each slot of sentenced_images (same index): deleted, but + /// counted in the driver's usage until the ring destroys them. + std::array<u64, TICKS_TO_DESTROY> sentenced_image_bytes{}; + size_t sentenced_image_index = 0; + /// Deleted images on runtimes that report GPU progress (CurrentTick, IsFree): destroyed, with + /// their device memory, as soon as the GPU is past them rather than TICKS_TO_DESTROY frames + /// later. While the collector deletes dozens a frame, eight frames of them held 1-1.5 GB of + /// VRAM above its threshold (title 1 on an 8 GB card: 6.6-7.0 GB in use against 5 GiB). + struct GpuSentencedImage { + u64 tick; + u64 bytes; + Image image; + }; + std::deque<GpuSentencedImage> gpu_sentenced_images; + u64 gpu_sentenced_bytes{}; + + /// Bytes of images per device memory block, for the collector (refreshed every 60 frames). + std::unordered_map<u64, u64> block_image_bytes; + u64 block_image_bytes_frame{~0ULL}; DelayedDestructionRing<ImageView, TICKS_TO_DESTROY> sentenced_image_view; DelayedDestructionRing<Framebuffer, TICKS_TO_DESTROY> sentenced_framebuffers; @@ -566,6 +617,8 @@ private: Common::ThreadWorker texture_decode_worker{1, "TextureDecoder", {}, Common::ThreadPlacement::Efficiency}; std::vector<std::unique_ptr<AsyncDecodeContext>> async_decodes; + /// Guest memory images were uploaded from since the last cold hint (device address, size). + std::vector<std::pair<DAddr, u64>> uploaded_ranges; std::deque<PendingUnswizzle> unswizzle_queue; u8 current_unswizzle_frame; diff --git a/src/video_core/texture_cache/types.h b/src/video_core/texture_cache/types.h index ecacd8c6bd..9e541d6afc 100644 --- a/src/video_core/texture_cache/types.h +++ b/src/video_core/texture_cache/types.h @@ -132,6 +132,15 @@ struct ImageCopy { Extent3D extent; }; +/// A rectangle of one mip level, in texels. +struct LevelRect { + s32 level; + u32 x; + u32 y; + u32 width; + u32 height; +}; + struct BufferImageCopy { size_t buffer_offset; size_t buffer_size; diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp index ef34019a08..0e1acba813 100644 --- a/src/video_core/texture_cache/util.cpp +++ b/src/video_core/texture_cache/util.cpp @@ -929,6 +929,210 @@ boost::container::small_vector<BufferImageCopy, 16> UnswizzleImage(Tegra::Memory return copies; } +namespace { +// One level of a single-layer 2D block-linear image of 1x1-texel blocks, laid out as +// UnswizzleImage reads it: rows of blocks, each block one GOB wide and 2^block_height GOBs tall. +struct BlockLinearLevel { + u32 offset; ///< Into the image + u32 width; ///< Texels + u32 height; + u32 block_height; ///< Log2 of the GOBs a block is tall + u32 block_texels; ///< Texels across a block + u32 block_rows; ///< Texel rows in a block + u32 block_bytes; + u32 blocks_per_row; +}; + +std::vector<BlockLinearLevel> BlockLinearLevels(const ImageInfo& info) { + const LevelInfo level_info = MakeLevelInfo(info); + const u32 bpp_log2 = BytesPerBlockLog2(info.format); + const Extent2D gob = GobSize(bpp_log2, info.block.height, info.tile_width_spacing); + std::vector<BlockLinearLevel> levels; + u32 offset = 0; + for (s32 level = 0; level < info.resources.levels; ++level) { + const Extent3D level_size = AdjustMipSize(info.size, level); + const Extent3D num_tiles = AdjustTileSize(level_size, level_info.tile_size); + const Extent3D block = + AdjustMipBlockSize(num_tiles, level_info.block, level, level_info.num_levels); + const u32 stride_alignment = StrideAlignment(num_tiles, info.block, gob, bpp_log2); + const u32 stride = Common::AlignUpLog2(num_tiles.width, stride_alignment) << bpp_log2; + levels.push_back({ + .offset = offset, + .width = level_size.width, + .height = level_size.height, + .block_height = block.height, + .block_texels = GOB_SIZE_X >> bpp_log2, + .block_rows = GOB_SIZE_Y << block.height, + .block_bytes = GOB_SIZE << block.height, + .blocks_per_row = Common::DivCeilLog2(stride, GOB_SIZE_X_SHIFT), + }); + offset += CalculateLevelSize(level_info, level); + } + return levels; +} + +// Rows of blocks a rectangle covers, and its blocks across. +struct RectBlocks { + u32 first_row; + u32 last_row; + u32 first_block; + u32 num_blocks; +}; + +RectBlocks BlocksOf(const BlockLinearLevel& level, const LevelRect& rect) { + const u32 first_block = rect.x / level.block_texels; + return { + .first_row = rect.y / level.block_rows, + .last_row = (rect.y + rect.height - 1) / level.block_rows, + .first_block = first_block, + .num_blocks = (std::min)(Common::DivCeil(rect.width, level.block_texels), + level.blocks_per_row - first_block), + }; +} + +u32 BlockRowTexelRows(const BlockLinearLevel& level, u32 row) { + return (std::min)(level.block_rows, level.height - row * level.block_rows); +} +} // Anonymous namespace + +std::vector<LevelRect> SparseTileRects(const ImageInfo& info, + std::span<const std::pair<u64, u64>> ranges, u32 tile_width, + u32 tile_height, s32 tail_level) { + std::vector<LevelRect> rects; + const std::vector<BlockLinearLevel> levels = BlockLinearLevels(info); + const u64 image_end = CalculateLevelBytes( + CalculateLevelSizes(MakeLevelInfo(info), info.resources.levels), info.resources.levels); + for (s32 index = 0; index < static_cast<s32>(levels.size()); ++index) { + const BlockLinearLevel& level = levels[index]; + const u64 level_end = + index + 1 < static_cast<s32>(levels.size()) ? levels[index + 1].offset : image_end; + const auto overlaps = [&](const std::pair<u64, u64>& range) { + return range.first < level_end && range.first + range.second > level.offset; + }; + if (index >= tail_level) { + if (std::ranges::any_of(ranges, overlaps)) { + rects.push_back({index, 0, 0, level.width, level.height}); + } + continue; + } + const u32 tiles_x = Common::DivCeil(level.width, tile_width); + const u32 tiles_y = Common::DivCeil(level.height, tile_height); + std::vector<bool> touched(size_t{tiles_x} * tiles_y); + for (const auto& range : ranges) { + if (!overlaps(range)) { + continue; + } + const u64 begin = (std::max<u64>)(range.first, level.offset) - level.offset; + const u64 end = (std::min<u64>)(range.first + range.second, level_end) - level.offset; + const u64 first = begin / level.block_bytes; + const u64 last = (end - 1) / level.block_bytes; + for (u64 row = first / level.blocks_per_row; row <= last / level.blocks_per_row; ++row) { + const u64 row_start = row * level.blocks_per_row; + const u64 low = (std::max)(first, row_start) - row_start; + const u64 high = (std::min)(last, row_start + level.blocks_per_row - 1) - row_start; + const u64 x0 = low * level.block_texels; + const u64 y0 = row * level.block_rows; + if (x0 >= level.width || y0 >= level.height) { + continue; + } + const u64 x1 = (std::min<u64>)((high + 1) * level.block_texels, level.width); + const u64 y1 = (std::min<u64>)(y0 + level.block_rows, level.height); + for (u64 ty = y0 / tile_height; ty <= (y1 - 1) / tile_height; ++ty) { + for (u64 tx = x0 / tile_width; tx <= (x1 - 1) / tile_width; ++tx) { + touched[ty * tiles_x + tx] = true; + } + } + } + } + for (u32 ty = 0; ty < tiles_y; ++ty) { + for (u32 tx = 0; tx < tiles_x;) { + if (!touched[size_t{ty} * tiles_x + tx]) { + ++tx; + continue; + } + const u32 first_tile = tx; + while (tx < tiles_x && touched[size_t{ty} * tiles_x + tx]) { + ++tx; + } + const u32 x = first_tile * tile_width; + const u32 y = ty * tile_height; + rects.push_back({ + .level = index, + .x = x, + .y = y, + .width = (std::min)(tx * tile_width, level.width) - x, + .height = (std::min)(y + tile_height, level.height) - y, + }); + } + } + } + return rects; +} + +size_t UnswizzledRectsBytes(const ImageInfo& info, std::span<const LevelRect> rects) { + const std::vector<BlockLinearLevel> levels = BlockLinearLevels(info); + const u32 bpp_log2 = BytesPerBlockLog2(info.format); + size_t bytes = 0; + for (const LevelRect& rect : rects) { + const BlockLinearLevel& level = levels[rect.level]; + const RectBlocks blocks = BlocksOf(level, rect); + for (u32 row = blocks.first_row; row <= blocks.last_row; ++row) { + bytes += (size_t{blocks.num_blocks * level.block_texels} << bpp_log2) * + BlockRowTexelRows(level, row); + } + } + return bytes; +} + +std::vector<BufferImageCopy> UnswizzleImageRects(Tegra::MemoryManager& gpu_memory, + GPUVAddr gpu_addr, const ImageInfo& info, + std::span<const LevelRect> rects, + std::span<u8> output) { + const std::vector<BlockLinearLevel> levels = BlockLinearLevels(info); + const u32 bpp_log2 = BytesPerBlockLog2(info.format); + std::vector<BufferImageCopy> copies; + std::vector<u8> swizzled; + size_t output_offset = 0; + for (const LevelRect& rect : rects) { + const BlockLinearLevel& level = levels[rect.level]; + const RectBlocks blocks = BlocksOf(level, rect); + const u32 width = blocks.num_blocks * level.block_texels; + const size_t row_bytes = size_t{blocks.num_blocks} * level.block_bytes; + for (u32 row = blocks.first_row; row <= blocks.last_row; ++row) { + // A row of whole blocks is laid out like an image of that width on its own. + const u32 texel_rows = BlockRowTexelRows(level, row); + const size_t bytes = (size_t{width} << bpp_log2) * texel_rows; + const GPUVAddr row_addr = gpu_addr + level.offset + + u64{row} * level.blocks_per_row * level.block_bytes + + u64{blocks.first_block} * level.block_bytes; + swizzled.assign(row_bytes, 0); + gpu_memory.ReadBlockUnsafe(row_addr, swizzled.data(), row_bytes); + UnswizzleTexture(output.subspan(output_offset, bytes), swizzled, 1U << bpp_log2, width, + texel_rows, 1, level.block_height, 0, 0, + gpu_memory.UsesNativeTiling()); + const u32 row_y = row * level.block_rows; + const u32 y0 = (std::max)(rect.y, row_y); + const u32 y1 = (std::min)(rect.y + rect.height, row_y + texel_rows); + copies.push_back(BufferImageCopy{ + .buffer_offset = output_offset + ((size_t{y0 - row_y} * width) << bpp_log2), + .buffer_size = bytes, + .buffer_row_length = width, + .buffer_image_height = texel_rows, + .image_subresource = + { + .base_level = rect.level, + .base_layer = 0, + .num_layers = 1, + }, + .image_offset = {static_cast<s32>(rect.x), static_cast<s32>(y0), 0}, + .image_extent = {rect.width, y1 - y0, 1}, + }); + output_offset += bytes; + } + } + return copies; +} + void ConvertedLayout(const ImageInfo& info, std::span<BufferImageCopy> copies) { // Same offsets and sizes as ConvertImage below. const auto recompression = Settings::GetAstcRecompression(); diff --git a/src/video_core/texture_cache/util.h b/src/video_core/texture_cache/util.h index 0381d2d24b..74da0f90f9 100644 --- a/src/video_core/texture_cache/util.h +++ b/src/video_core/texture_cache/util.h @@ -8,6 +8,8 @@ #include <optional> #include <span> +#include <utility> +#include <vector> #include <boost/container/small_vector.hpp> #include "common/common_types.h" @@ -72,6 +74,25 @@ struct OverlapResult { Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const ImageInfo& info, std::span<const u8> input, std::span<u8> output); +/// For a single-layer 2D block-linear image of 1x1-texel blocks: the host sparse blocks +/// (tile_width x tile_height texels) that the guest byte ranges (offset, size) touch, merged into +/// runs along x. Levels from tail_level on (the host's mip tail) come whole. +[[nodiscard]] std::vector<LevelRect> SparseTileRects(const ImageInfo& info, + std::span<const std::pair<u64, u64>> ranges, + u32 tile_width, u32 tile_height, + s32 tail_level); + +/// Bytes UnswizzleImageRects writes for `rects`. +[[nodiscard]] size_t UnswizzledRectsBytes(const ImageInfo& info, std::span<const LevelRect> rects); + +/// Unswizzles `rects` (from SparseTileRects: on GOB columns) of a single-layer 2D block-linear +/// image into output, reading only the guest blocks under them. +[[nodiscard]] std::vector<BufferImageCopy> UnswizzleImageRects(Tegra::MemoryManager& gpu_memory, + GPUVAddr gpu_addr, + const ImageInfo& info, + std::span<const LevelRect> rects, + std::span<u8> output); + /// Rewrites `copies` (from UnswizzleImage) to the layout ConvertImage produces, without converting. void ConvertedLayout(const ImageInfo& info, std::span<BufferImageCopy> copies); |