From 64c9a90c20e73b66338afb1d8cad7631b033b908 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sun, 20 Nov 2022 00:09:56 +0100 Subject: Buffer Cache: Fully rework the buffer cache. --- src/video_core/buffer_cache/buffer_base.h | 459 ++---------------------------- 1 file changed, 25 insertions(+), 434 deletions(-) (limited to 'src/video_core/buffer_cache/buffer_base.h') diff --git a/src/video_core/buffer_cache/buffer_base.h b/src/video_core/buffer_cache/buffer_base.h index 1b4d63616d..66d8bb43cb 100644 --- a/src/video_core/buffer_cache/buffer_base.h +++ b/src/video_core/buffer_cache/buffer_base.h @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later +// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later #pragma once @@ -11,9 +11,7 @@ #include "common/alignment.h" #include "common/common_funcs.h" #include "common/common_types.h" -#include "common/div_ceil.h" -#include "common/settings.h" -#include "core/memory.h" +#include "video_core/buffer_cache/word_manager.h" namespace VideoCommon { @@ -36,116 +34,14 @@ struct NullBufferParams {}; */ template class BufferBase { - static constexpr u64 PAGES_PER_WORD = 64; - static constexpr u64 BYTES_PER_PAGE = Core::Memory::YUZU_PAGESIZE; - static constexpr u64 BYTES_PER_WORD = PAGES_PER_WORD * BYTES_PER_PAGE; - - /// Vector tracking modified pages tightly packed with small vector optimization - union WordsArray { - /// Returns the pointer to the words state - [[nodiscard]] const u64* Pointer(bool is_short) const noexcept { - return is_short ? &stack : heap; - } - - /// Returns the pointer to the words state - [[nodiscard]] u64* Pointer(bool is_short) noexcept { - return is_short ? &stack : heap; - } - - u64 stack = 0; ///< Small buffers storage - u64* heap; ///< Not-small buffers pointer to the storage - }; - - struct Words { - explicit Words() = default; - explicit Words(u64 size_bytes_) : size_bytes{size_bytes_} { - if (IsShort()) { - cpu.stack = ~u64{0}; - gpu.stack = 0; - cached_cpu.stack = 0; - untracked.stack = ~u64{0}; - } else { - // Share allocation between CPU and GPU pages and set their default values - const size_t num_words = NumWords(); - u64* const alloc = new u64[num_words * 4]; - cpu.heap = alloc; - gpu.heap = alloc + num_words; - cached_cpu.heap = alloc + num_words * 2; - untracked.heap = alloc + num_words * 3; - std::fill_n(cpu.heap, num_words, ~u64{0}); - std::fill_n(gpu.heap, num_words, 0); - std::fill_n(cached_cpu.heap, num_words, 0); - std::fill_n(untracked.heap, num_words, ~u64{0}); - } - // Clean up tailing bits - const u64 last_word_size = size_bytes % BYTES_PER_WORD; - const u64 last_local_page = Common::DivCeil(last_word_size, BYTES_PER_PAGE); - const u64 shift = (PAGES_PER_WORD - last_local_page) % PAGES_PER_WORD; - const u64 last_word = (~u64{0} << shift) >> shift; - cpu.Pointer(IsShort())[NumWords() - 1] = last_word; - untracked.Pointer(IsShort())[NumWords() - 1] = last_word; - } - - ~Words() { - Release(); - } - - Words& operator=(Words&& rhs) noexcept { - Release(); - size_bytes = rhs.size_bytes; - cpu = rhs.cpu; - gpu = rhs.gpu; - cached_cpu = rhs.cached_cpu; - untracked = rhs.untracked; - rhs.cpu.heap = nullptr; - return *this; - } - - Words(Words&& rhs) noexcept - : size_bytes{rhs.size_bytes}, cpu{rhs.cpu}, gpu{rhs.gpu}, - cached_cpu{rhs.cached_cpu}, untracked{rhs.untracked} { - rhs.cpu.heap = nullptr; - } - - Words& operator=(const Words&) = delete; - Words(const Words&) = delete; - - /// Returns true when the buffer fits in the small vector optimization - [[nodiscard]] bool IsShort() const noexcept { - return size_bytes <= BYTES_PER_WORD; - } - - /// Returns the number of words of the buffer - [[nodiscard]] size_t NumWords() const noexcept { - return Common::DivCeil(size_bytes, BYTES_PER_WORD); - } - - /// Release buffer resources - void Release() { - if (!IsShort()) { - // CPU written words is the base for the heap allocation - delete[] cpu.heap; - } - } - - u64 size_bytes = 0; - WordsArray cpu; - WordsArray gpu; - WordsArray cached_cpu; - WordsArray untracked; - }; - - enum class Type { - CPU, - GPU, - CachedCPU, - Untracked, - }; - public: + static constexpr u64 BASE_PAGE_BITS = 16; + static constexpr u64 BASE_PAGE_SIZE = 1ULL << BASE_PAGE_BITS; + explicit BufferBase(RasterizerInterface& rasterizer_, VAddr cpu_addr_, u64 size_bytes) - : rasterizer{&rasterizer_}, cpu_addr{Common::AlignDown(cpu_addr_, BYTES_PER_PAGE)}, - words(Common::AlignUp(size_bytes + (cpu_addr_ - cpu_addr), BYTES_PER_PAGE)) {} + : cpu_addr{Common::AlignDown(cpu_addr_, BASE_PAGE_SIZE)}, + word_manager(cpu_addr, rasterizer_, + Common::AlignUp(size_bytes + (cpu_addr_ - cpu_addr), BASE_PAGE_SIZE)) {} explicit BufferBase(NullBufferParams) {} @@ -159,94 +55,82 @@ public: [[nodiscard]] std::pair ModifiedCpuRegion(VAddr query_cpu_addr, u64 query_size) const noexcept { const u64 offset = query_cpu_addr - cpu_addr; - return ModifiedRegion(offset, query_size); + return word_manager.ModifiedRegion(offset, query_size); } /// Returns the inclusive GPU modified range in a begin end pair [[nodiscard]] std::pair ModifiedGpuRegion(VAddr query_cpu_addr, u64 query_size) const noexcept { const u64 offset = query_cpu_addr - cpu_addr; - return ModifiedRegion(offset, query_size); + return word_manager.ModifiedRegion(offset, query_size); } /// Returns true if a region has been modified from the CPU [[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) const noexcept { const u64 offset = query_cpu_addr - cpu_addr; - return IsRegionModified(offset, query_size); + return word_manager.IsRegionModified(offset, query_size); } /// Returns true if a region has been modified from the GPU [[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) const noexcept { const u64 offset = query_cpu_addr - cpu_addr; - return IsRegionModified(offset, query_size); + return word_manager.IsRegionModified(offset, query_size); } /// Mark region as CPU modified, notifying the rasterizer about this change void MarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 size) { - ChangeRegionState(dirty_cpu_addr, size); + word_manager.ChangeRegionState(dirty_cpu_addr, size); } /// Unmark region as CPU modified, notifying the rasterizer about this change void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 size) { - ChangeRegionState(dirty_cpu_addr, size); + word_manager.ChangeRegionState(dirty_cpu_addr, size); } /// Mark region as modified from the host GPU void MarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 size) noexcept { - ChangeRegionState(dirty_cpu_addr, size); + word_manager.ChangeRegionState(dirty_cpu_addr, size); } /// Unmark region as modified from the host GPU void UnmarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 size) noexcept { - ChangeRegionState(dirty_cpu_addr, size); + word_manager.ChangeRegionState(dirty_cpu_addr, size); } /// Mark region as modified from the CPU /// but don't mark it as modified until FlusHCachedWrites is called. void CachedCpuWrite(VAddr dirty_cpu_addr, u64 size) { flags |= BufferFlagBits::CachedWrites; - ChangeRegionState(dirty_cpu_addr, size); + word_manager.ChangeRegionState(dirty_cpu_addr, size); } /// Flushes cached CPU writes, and notify the rasterizer about the deltas void FlushCachedWrites() noexcept { flags &= ~BufferFlagBits::CachedWrites; - const u64 num_words = NumWords(); - u64* const cached_words = Array(); - u64* const untracked_words = Array(); - u64* const cpu_words = Array(); - for (u64 word_index = 0; word_index < num_words; ++word_index) { - const u64 cached_bits = cached_words[word_index]; - NotifyRasterizer(word_index, untracked_words[word_index], cached_bits); - untracked_words[word_index] |= cached_bits; - cpu_words[word_index] |= cached_bits; - if (!Settings::values.use_pessimistic_flushes) { - cached_words[word_index] = 0; - } - } + word_manager.FlushCachedWrites(); } /// Call 'func' for each CPU modified range and unmark those pages as CPU modified template void ForEachUploadRange(VAddr query_cpu_range, u64 size, Func&& func) { - ForEachModifiedRange(query_cpu_range, size, true, func); + word_manager.ForEachModifiedRange(query_cpu_range, size, true, func); } /// Call 'func' for each GPU modified range and unmark those pages as GPU modified template void ForEachDownloadRange(VAddr query_cpu_range, u64 size, bool clear, Func&& func) { - ForEachModifiedRange(query_cpu_range, size, clear, func); + word_manager.ForEachModifiedRange(query_cpu_range, size, clear, func); } template void ForEachDownloadRangeAndClear(VAddr query_cpu_range, u64 size, Func&& func) { - ForEachModifiedRange(query_cpu_range, size, true, func); + word_manager.ForEachModifiedRange(query_cpu_range, size, true, func); } /// Call 'func' for each GPU modified range and unmark those pages as GPU modified template void ForEachDownloadRange(Func&& func) { - ForEachModifiedRange(cpu_addr, SizeBytes(), true, func); + word_manager.ForEachModifiedRange(cpu_addr, SizeBytes(), true, func); } /// Mark buffer as picked @@ -297,7 +181,7 @@ public: /// Returns the size in bytes of the buffer [[nodiscard]] u64 SizeBytes() const noexcept { - return words.size_bytes; + return word_manager.SizeBytes(); } size_t getLRUID() const noexcept { @@ -309,301 +193,8 @@ public: } private: - template - u64* Array() noexcept { - if constexpr (type == Type::CPU) { - return words.cpu.Pointer(IsShort()); - } else if constexpr (type == Type::GPU) { - return words.gpu.Pointer(IsShort()); - } else if constexpr (type == Type::CachedCPU) { - return words.cached_cpu.Pointer(IsShort()); - } else if constexpr (type == Type::Untracked) { - return words.untracked.Pointer(IsShort()); - } - } - - template - const u64* Array() const noexcept { - if constexpr (type == Type::CPU) { - return words.cpu.Pointer(IsShort()); - } else if constexpr (type == Type::GPU) { - return words.gpu.Pointer(IsShort()); - } else if constexpr (type == Type::CachedCPU) { - return words.cached_cpu.Pointer(IsShort()); - } else if constexpr (type == Type::Untracked) { - return words.untracked.Pointer(IsShort()); - } - } - - /** - * Change the state of a range of pages - * - * @param dirty_addr Base address to mark or unmark as modified - * @param size Size in bytes to mark or unmark as modified - */ - template - void ChangeRegionState(u64 dirty_addr, s64 size) noexcept(type == Type::GPU) { - const s64 difference = dirty_addr - cpu_addr; - const u64 offset = std::max(difference, 0); - size += std::min(difference, 0); - if (offset >= SizeBytes() || size < 0) { - return; - } - u64* const untracked_words = Array(); - u64* const state_words = Array(); - const u64 offset_end = std::min(offset + size, SizeBytes()); - const u64 begin_page_index = offset / BYTES_PER_PAGE; - const u64 begin_word_index = begin_page_index / PAGES_PER_WORD; - const u64 end_page_index = Common::DivCeil(offset_end, BYTES_PER_PAGE); - const u64 end_word_index = Common::DivCeil(end_page_index, PAGES_PER_WORD); - u64 page_index = begin_page_index % PAGES_PER_WORD; - u64 word_index = begin_word_index; - while (word_index < end_word_index) { - const u64 next_word_first_page = (word_index + 1) * PAGES_PER_WORD; - const u64 left_offset = - std::min(next_word_first_page - end_page_index, PAGES_PER_WORD) % PAGES_PER_WORD; - const u64 right_offset = page_index; - u64 bits = ~u64{0}; - bits = (bits >> right_offset) << right_offset; - bits = (bits << left_offset) >> left_offset; - if constexpr (type == Type::CPU || type == Type::CachedCPU) { - NotifyRasterizer(word_index, untracked_words[word_index], bits); - } - if constexpr (enable) { - state_words[word_index] |= bits; - if constexpr (type == Type::CPU || type == Type::CachedCPU) { - untracked_words[word_index] |= bits; - } - } else { - state_words[word_index] &= ~bits; - if constexpr (type == Type::CPU || type == Type::CachedCPU) { - untracked_words[word_index] &= ~bits; - } - } - page_index = 0; - ++word_index; - } - } - - /** - * Notify rasterizer about changes in the CPU tracking state of a word in the buffer - * - * @param word_index Index to the word to notify to the rasterizer - * @param current_bits Current state of the word - * @param new_bits New state of the word - * - * @tparam add_to_rasterizer True when the rasterizer should start tracking the new pages - */ - template - void NotifyRasterizer(u64 word_index, u64 current_bits, u64 new_bits) const { - u64 changed_bits = (add_to_rasterizer ? current_bits : ~current_bits) & new_bits; - VAddr addr = cpu_addr + word_index * BYTES_PER_WORD; - while (changed_bits != 0) { - const int empty_bits = std::countr_zero(changed_bits); - addr += empty_bits * BYTES_PER_PAGE; - changed_bits >>= empty_bits; - - const u32 continuous_bits = std::countr_one(changed_bits); - const u64 size = continuous_bits * BYTES_PER_PAGE; - const VAddr begin_addr = addr; - addr += size; - changed_bits = continuous_bits < PAGES_PER_WORD ? (changed_bits >> continuous_bits) : 0; - rasterizer->UpdatePagesCachedCount(begin_addr, size, add_to_rasterizer ? 1 : -1); - } - } - - /** - * Loop over each page in the given range, turn off those bits and notify the rasterizer if - * needed. Call the given function on each turned off range. - * - * @param query_cpu_range Base CPU address to loop over - * @param size Size in bytes of the CPU range to loop over - * @param func Function to call for each turned off region - */ - template - void ForEachModifiedRange(VAddr query_cpu_range, s64 size, bool clear, Func&& func) { - static_assert(type != Type::Untracked); - - const s64 difference = query_cpu_range - cpu_addr; - const u64 query_begin = std::max(difference, 0); - size += std::min(difference, 0); - if (query_begin >= SizeBytes() || size < 0) { - return; - } - u64* const untracked_words = Array(); - u64* const state_words = Array(); - const u64 query_end = query_begin + std::min(static_cast(size), SizeBytes()); - u64* const words_begin = state_words + query_begin / BYTES_PER_WORD; - u64* const words_end = state_words + Common::DivCeil(query_end, BYTES_PER_WORD); - - const auto modified = [](u64 word) { return word != 0; }; - const auto first_modified_word = std::find_if(words_begin, words_end, modified); - if (first_modified_word == words_end) { - // Exit early when the buffer is not modified - return; - } - const auto last_modified_word = std::find_if_not(first_modified_word, words_end, modified); - - const u64 word_index_begin = std::distance(state_words, first_modified_word); - const u64 word_index_end = std::distance(state_words, last_modified_word); - - const unsigned local_page_begin = std::countr_zero(*first_modified_word); - const unsigned local_page_end = - static_cast(PAGES_PER_WORD) - std::countl_zero(last_modified_word[-1]); - const u64 word_page_begin = word_index_begin * PAGES_PER_WORD; - const u64 word_page_end = (word_index_end - 1) * PAGES_PER_WORD; - const u64 query_page_begin = query_begin / BYTES_PER_PAGE; - const u64 query_page_end = Common::DivCeil(query_end, BYTES_PER_PAGE); - const u64 page_index_begin = std::max(word_page_begin + local_page_begin, query_page_begin); - const u64 page_index_end = std::min(word_page_end + local_page_end, query_page_end); - const u64 first_word_page_begin = page_index_begin % PAGES_PER_WORD; - const u64 last_word_page_end = (page_index_end - 1) % PAGES_PER_WORD + 1; - - u64 page_begin = first_word_page_begin; - u64 current_base = 0; - u64 current_size = 0; - bool on_going = false; - for (u64 word_index = word_index_begin; word_index < word_index_end; ++word_index) { - const bool is_last_word = word_index + 1 == word_index_end; - const u64 page_end = is_last_word ? last_word_page_end : PAGES_PER_WORD; - const u64 right_offset = page_begin; - const u64 left_offset = PAGES_PER_WORD - page_end; - u64 bits = ~u64{0}; - bits = (bits >> right_offset) << right_offset; - bits = (bits << left_offset) >> left_offset; - - const u64 current_word = state_words[word_index] & bits; - if (clear) { - state_words[word_index] &= ~bits; - } - - if constexpr (type == Type::CPU) { - const u64 current_bits = untracked_words[word_index] & bits; - untracked_words[word_index] &= ~bits; - NotifyRasterizer(word_index, current_bits, ~u64{0}); - } - // Exclude CPU modified pages when visiting GPU pages - const u64 word = current_word & ~(type == Type::GPU ? untracked_words[word_index] : 0); - u64 page = page_begin; - page_begin = 0; - - while (page < page_end) { - const int empty_bits = std::countr_zero(word >> page); - if (on_going && empty_bits != 0) { - InvokeModifiedRange(func, current_size, current_base); - current_size = 0; - on_going = false; - } - if (empty_bits == PAGES_PER_WORD) { - break; - } - page += empty_bits; - - const int continuous_bits = std::countr_one(word >> page); - if (!on_going && continuous_bits != 0) { - current_base = word_index * PAGES_PER_WORD + page; - on_going = true; - } - current_size += continuous_bits; - page += continuous_bits; - } - } - if (on_going && current_size > 0) { - InvokeModifiedRange(func, current_size, current_base); - } - } - - template - void InvokeModifiedRange(Func&& func, u64 current_size, u64 current_base) { - const u64 current_size_bytes = current_size * BYTES_PER_PAGE; - const u64 offset_begin = current_base * BYTES_PER_PAGE; - const u64 offset_end = std::min(offset_begin + current_size_bytes, SizeBytes()); - func(offset_begin, offset_end - offset_begin); - } - - /** - * Returns true when a region has been modified - * - * @param offset Offset in bytes from the start of the buffer - * @param size Size in bytes of the region to query for modifications - */ - template - [[nodiscard]] bool IsRegionModified(u64 offset, u64 size) const noexcept { - static_assert(type != Type::Untracked); - - const u64* const untracked_words = Array(); - const u64* const state_words = Array(); - const u64 num_query_words = size / BYTES_PER_WORD + 1; - const u64 word_begin = offset / BYTES_PER_WORD; - const u64 word_end = std::min(word_begin + num_query_words, NumWords()); - const u64 page_limit = Common::DivCeil(offset + size, BYTES_PER_PAGE); - u64 page_index = (offset / BYTES_PER_PAGE) % PAGES_PER_WORD; - for (u64 word_index = word_begin; word_index < word_end; ++word_index, page_index = 0) { - const u64 off_word = type == Type::GPU ? untracked_words[word_index] : 0; - const u64 word = state_words[word_index] & ~off_word; - if (word == 0) { - continue; - } - const u64 page_end = std::min((word_index + 1) * PAGES_PER_WORD, page_limit); - const u64 local_page_end = page_end % PAGES_PER_WORD; - const u64 page_end_shift = (PAGES_PER_WORD - local_page_end) % PAGES_PER_WORD; - if (((word >> page_index) << page_index) << page_end_shift != 0) { - return true; - } - } - return false; - } - - /** - * Returns a begin end pair with the inclusive modified region - * - * @param offset Offset in bytes from the start of the buffer - * @param size Size in bytes of the region to query for modifications - */ - template - [[nodiscard]] std::pair ModifiedRegion(u64 offset, u64 size) const noexcept { - static_assert(type != Type::Untracked); - - const u64* const untracked_words = Array(); - const u64* const state_words = Array(); - const u64 num_query_words = size / BYTES_PER_WORD + 1; - const u64 word_begin = offset / BYTES_PER_WORD; - const u64 word_end = std::min(word_begin + num_query_words, NumWords()); - const u64 page_base = offset / BYTES_PER_PAGE; - const u64 page_limit = Common::DivCeil(offset + size, BYTES_PER_PAGE); - u64 begin = std::numeric_limits::max(); - u64 end = 0; - for (u64 word_index = word_begin; word_index < word_end; ++word_index) { - const u64 off_word = type == Type::GPU ? untracked_words[word_index] : 0; - const u64 word = state_words[word_index] & ~off_word; - if (word == 0) { - continue; - } - const u64 local_page_begin = std::countr_zero(word); - const u64 local_page_end = PAGES_PER_WORD - std::countl_zero(word); - const u64 page_index = word_index * PAGES_PER_WORD; - const u64 page_begin = std::max(page_index + local_page_begin, page_base); - const u64 page_end = std::min(page_index + local_page_end, page_limit); - begin = std::min(begin, page_begin); - end = std::max(end, page_end); - } - static constexpr std::pair EMPTY{0, 0}; - return begin < end ? std::make_pair(begin * BYTES_PER_PAGE, end * BYTES_PER_PAGE) : EMPTY; - } - - /// Returns the number of words of the buffer - [[nodiscard]] size_t NumWords() const noexcept { - return words.NumWords(); - } - - /// Returns true when the buffer fits in the small vector optimization - [[nodiscard]] bool IsShort() const noexcept { - return words.IsShort(); - } - - RasterizerInterface* rasterizer = nullptr; VAddr cpu_addr = 0; - Words words; + WordManager word_manager; BufferFlagBits flags{}; int stream_score = 0; size_t lru_id = SIZE_MAX; -- cgit v1.2.3 From 917a21317f7b8c46a6528b4f660757134d149855 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sun, 23 Apr 2023 03:58:16 +0200 Subject: Clang: format and ficx compile errors. --- src/video_core/buffer_cache/buffer_base.h | 26 +++---- src/video_core/buffer_cache/buffer_cache.h | 4 +- src/video_core/buffer_cache/buffer_cache_base.h | 32 ++++----- src/video_core/buffer_cache/memory_tracker_base.h | 82 +++++++++++++---------- src/video_core/renderer_vulkan/vk_buffer_cache.h | 2 +- 5 files changed, 78 insertions(+), 68 deletions(-) (limited to 'src/video_core/buffer_cache/buffer_base.h') diff --git a/src/video_core/buffer_cache/buffer_base.h b/src/video_core/buffer_cache/buffer_base.h index 66d8bb43cb..095f793878 100644 --- a/src/video_core/buffer_cache/buffer_base.h +++ b/src/video_core/buffer_cache/buffer_base.h @@ -55,53 +55,53 @@ public: [[nodiscard]] std::pair ModifiedCpuRegion(VAddr query_cpu_addr, u64 query_size) const noexcept { const u64 offset = query_cpu_addr - cpu_addr; - return word_manager.ModifiedRegion(offset, query_size); + return word_manager.template ModifiedRegion(offset, query_size); } /// Returns the inclusive GPU modified range in a begin end pair [[nodiscard]] std::pair ModifiedGpuRegion(VAddr query_cpu_addr, u64 query_size) const noexcept { const u64 offset = query_cpu_addr - cpu_addr; - return word_manager.ModifiedRegion(offset, query_size); + return word_manager.template ModifiedRegion(offset, query_size); } /// Returns true if a region has been modified from the CPU [[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) const noexcept { const u64 offset = query_cpu_addr - cpu_addr; - return word_manager.IsRegionModified(offset, query_size); + return word_manager.template IsRegionModified(offset, query_size); } /// Returns true if a region has been modified from the GPU [[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) const noexcept { const u64 offset = query_cpu_addr - cpu_addr; - return word_manager.IsRegionModified(offset, query_size); + return word_manager.template IsRegionModified(offset, query_size); } /// Mark region as CPU modified, notifying the rasterizer about this change void MarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 size) { - word_manager.ChangeRegionState(dirty_cpu_addr, size); + word_manager.template ChangeRegionState(dirty_cpu_addr, size); } /// Unmark region as CPU modified, notifying the rasterizer about this change void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 size) { - word_manager.ChangeRegionState(dirty_cpu_addr, size); + word_manager.template ChangeRegionState(dirty_cpu_addr, size); } /// Mark region as modified from the host GPU void MarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 size) noexcept { - word_manager.ChangeRegionState(dirty_cpu_addr, size); + word_manager.template ChangeRegionState(dirty_cpu_addr, size); } /// Unmark region as modified from the host GPU void UnmarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 size) noexcept { - word_manager.ChangeRegionState(dirty_cpu_addr, size); + word_manager.template ChangeRegionState(dirty_cpu_addr, size); } /// Mark region as modified from the CPU /// but don't mark it as modified until FlusHCachedWrites is called. void CachedCpuWrite(VAddr dirty_cpu_addr, u64 size) { flags |= BufferFlagBits::CachedWrites; - word_manager.ChangeRegionState(dirty_cpu_addr, size); + word_manager.template ChangeRegionState(dirty_cpu_addr, size); } /// Flushes cached CPU writes, and notify the rasterizer about the deltas @@ -113,24 +113,24 @@ public: /// Call 'func' for each CPU modified range and unmark those pages as CPU modified template void ForEachUploadRange(VAddr query_cpu_range, u64 size, Func&& func) { - word_manager.ForEachModifiedRange(query_cpu_range, size, true, func); + word_manager.template ForEachModifiedRange(query_cpu_range, size, true, func); } /// Call 'func' for each GPU modified range and unmark those pages as GPU modified template void ForEachDownloadRange(VAddr query_cpu_range, u64 size, bool clear, Func&& func) { - word_manager.ForEachModifiedRange(query_cpu_range, size, clear, func); + word_manager.template ForEachModifiedRange(query_cpu_range, size, clear, func); } template void ForEachDownloadRangeAndClear(VAddr query_cpu_range, u64 size, Func&& func) { - word_manager.ForEachModifiedRange(query_cpu_range, size, true, func); + word_manager.template ForEachModifiedRange(query_cpu_range, size, true, func); } /// Call 'func' for each GPU modified range and unmark those pages as GPU modified template void ForEachDownloadRange(Func&& func) { - word_manager.ForEachModifiedRange(cpu_addr, SizeBytes(), true, func); + word_manager.template ForEachModifiedRange(cpu_addr, SizeBytes(), true, func); } /// Mark buffer as picked diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index faa48a678a..8fed08dab9 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -1395,10 +1395,10 @@ bool BufferCache

::SynchronizeBufferNoModified(Buffer& buffer, VAddr cpu_addr, auto make_copies = [&] { for (auto& interval : found_sets) { const std::size_t sub_size = interval.upper() - interval.lower(); - const VAddr cpu_addr = interval.lower(); + const VAddr cpu_addr_ = interval.lower(); copies.push_back(BufferCopy{ .src_offset = total_size_bytes, - .dst_offset = cpu_addr - buffer.CpuAddr(), + .dst_offset = cpu_addr_ - buffer.CpuAddr(), .size = sub_size, }); total_size_bytes += sub_size; diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index d4914a8f5c..acff22d4f4 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -16,10 +17,13 @@ #define BOOST_NO_MT #include #undef BOOST_NO_MT +#include +#include #include #include #include #include +#include #include "common/common_types.h" #include "common/div_ceil.h" @@ -42,7 +46,6 @@ #include "video_core/texture_cache/slot_vector.h" #include "video_core/texture_cache/types.h" - namespace boost { template class fast_pool_allocator; @@ -116,11 +119,10 @@ class BufferCache : public VideoCommon::ChannelSetupCaches; + using IntervalCompare = std::less; + using IntervalInstance = boost::icl::interval_type_default; + using IntervalAllocator = boost::fast_pool_allocator; + using IntervalSet = boost::icl::interval_set; using IntervalType = typename IntervalSet::interval_type; template @@ -141,12 +143,9 @@ class BufferCache : public VideoCommon::ChannelSetupCaches; + using OverlapCombine = counter_add_functor; + using OverlapSection = boost::icl::inter_section; + using OverlapCounter = boost::icl::split_interval_map; struct Empty {}; @@ -262,7 +261,8 @@ public: /// Return true when a CPU region is modified from the CPU [[nodiscard]] bool IsRegionCpuModified(VAddr addr, size_t size); - void SetDrawIndirect(const Tegra::Engines::DrawManager::IndirectParams* current_draw_indirect_) { + void SetDrawIndirect( + const Tegra::Engines::DrawManager::IndirectParams* current_draw_indirect_) { current_draw_indirect = current_draw_indirect_; } @@ -349,7 +349,8 @@ private: } } - void RemoveEachInOverlapCounter(OverlapCounter& current_range, const IntervalType search_interval, int subtract_value) { + void RemoveEachInOverlapCounter(OverlapCounter& current_range, + const IntervalType search_interval, int subtract_value) { bool any_removals = false; current_range.add(std::make_pair(search_interval, subtract_value)); do { @@ -469,7 +470,8 @@ private: void NotifyBufferDeletion(); - [[nodiscard]] Binding StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index, bool is_written) const; + [[nodiscard]] Binding StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index, + bool is_written) const; [[nodiscard]] TextureBufferBinding GetTextureBufferBinding(GPUVAddr gpu_addr, u32 size, PixelFormat format); diff --git a/src/video_core/buffer_cache/memory_tracker_base.h b/src/video_core/buffer_cache/memory_tracker_base.h index 93bd779c97..016d8430fc 100644 --- a/src/video_core/buffer_cache/memory_tracker_base.h +++ b/src/video_core/buffer_cache/memory_tracker_base.h @@ -35,67 +35,71 @@ public: /// Returns the inclusive CPU modified range in a begin end pair [[nodiscard]] std::pair ModifiedCpuRegion(VAddr query_cpu_addr, u64 query_size) noexcept { - return IteratePairs(query_cpu_addr, query_size, - [](Manager* manager, u64 offset, size_t size) { - return manager->ModifiedRegion(offset, size); - }); + return IteratePairs( + query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) { + return manager->template ModifiedRegion(offset, size); + }); } /// Returns the inclusive GPU modified range in a begin end pair [[nodiscard]] std::pair ModifiedGpuRegion(VAddr query_cpu_addr, u64 query_size) noexcept { - return IteratePairs(query_cpu_addr, query_size, - [](Manager* manager, u64 offset, size_t size) { - return manager->ModifiedRegion(offset, size); - }); + return IteratePairs( + query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) { + return manager->template ModifiedRegion(offset, size); + }); } /// Returns true if a region has been modified from the CPU [[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) noexcept { - return IteratePages(query_cpu_addr, query_size, - [](Manager* manager, u64 offset, size_t size) { - return manager->IsRegionModified(offset, size); - }); + return IteratePages( + query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) { + return manager->template IsRegionModified(offset, size); + }); } /// Returns true if a region has been modified from the GPU [[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) noexcept { - return IteratePages(query_cpu_addr, query_size, - [](Manager* manager, u64 offset, size_t size) { - return manager->IsRegionModified(offset, size); - }); + return IteratePages( + query_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) { + return manager->template IsRegionModified(offset, size); + }); } /// Mark region as CPU modified, notifying the rasterizer about this change void MarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) { - IteratePages( - dirty_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) { - manager->ChangeRegionState(manager->GetCpuAddr() + offset, size); - }); + IteratePages(dirty_cpu_addr, query_size, + [](Manager* manager, u64 offset, size_t size) { + manager->template ChangeRegionState( + manager->GetCpuAddr() + offset, size); + }); } /// Unmark region as CPU modified, notifying the rasterizer about this change void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) { - IteratePages( - dirty_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) { - manager->ChangeRegionState(manager->GetCpuAddr() + offset, size); - }); + IteratePages(dirty_cpu_addr, query_size, + [](Manager* manager, u64 offset, size_t size) { + manager->template ChangeRegionState( + manager->GetCpuAddr() + offset, size); + }); } /// Mark region as modified from the host GPU void MarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept { - IteratePages( - dirty_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) { - manager->ChangeRegionState(manager->GetCpuAddr() + offset, size); - }); + IteratePages(dirty_cpu_addr, query_size, + [](Manager* manager, u64 offset, size_t size) { + manager->template ChangeRegionState( + manager->GetCpuAddr() + offset, size); + }); } /// Unmark region as modified from the host GPU void UnmarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept { - IteratePages( - dirty_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) { - manager->ChangeRegionState(manager->GetCpuAddr() + offset, size); - }); + IteratePages(dirty_cpu_addr, query_size, + [](Manager* manager, u64 offset, size_t size) { + manager->template ChangeRegionState( + manager->GetCpuAddr() + offset, size); + }); } /// Mark region as modified from the CPU @@ -104,7 +108,7 @@ public: IteratePages( dirty_cpu_addr, query_size, [this](Manager* manager, u64 offset, size_t size) { const VAddr cpu_address = manager->GetCpuAddr() + offset; - manager->ChangeRegionState(cpu_address, size); + manager->template ChangeRegionState(cpu_address, size); cached_pages.insert(static_cast(cpu_address >> HIGHER_PAGE_BITS)); }); } @@ -128,7 +132,7 @@ public: void ForEachUploadRange(VAddr query_cpu_range, u64 query_size, Func&& func) { IteratePages(query_cpu_range, query_size, [&func](Manager* manager, u64 offset, size_t size) { - manager->ForEachModifiedRange( + manager->template ForEachModifiedRange( manager->GetCpuAddr() + offset, size, true, func); }); } @@ -138,7 +142,7 @@ public: void ForEachDownloadRange(VAddr query_cpu_range, u64 query_size, bool clear, Func&& func) { IteratePages(query_cpu_range, query_size, [&func, clear](Manager* manager, u64 offset, size_t size) { - manager->ForEachModifiedRange( + manager->template ForEachModifiedRange( manager->GetCpuAddr() + offset, size, clear, func); }); } @@ -147,7 +151,7 @@ public: void ForEachDownloadRangeAndClear(VAddr query_cpu_range, u64 query_size, Func&& func) { IteratePages(query_cpu_range, query_size, [&func](Manager* manager, u64 offset, size_t size) { - manager->ForEachModifiedRange( + manager->template ForEachModifiedRange( manager->GetCpuAddr() + offset, size, true, func); }); } @@ -218,7 +222,11 @@ private: page_offset = 0; remaining_size -= copy_amount; } - return begin < end ? std::make_pair(begin, end) : std::make_pair(0ULL, 0ULL); + if (begin < end) { + return std::make_pair(begin, end); + } else { + return std::make_pair(0ULL, 0ULL); + } } void CreateRegion(std::size_t page_index) { diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index 05968e6a63..879f1ed94d 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -3,7 +3,7 @@ #pragma once -#include "video_core/buffer_cache/buffer_cache.h" +#include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/buffer_cache/memory_tracker_base.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/renderer_vulkan/vk_compute_pass.h" -- cgit v1.2.3 From bd8abfe6543822147d3a834d20dcce9f1d992306 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Fri, 28 Apr 2023 23:54:54 +0200 Subject: BufferCache: Fixes and address feedback --- src/tests/video_core/memory_tracker.cpp | 4 +- src/video_core/buffer_cache/buffer_base.h | 99 +----- src/video_core/buffer_cache/buffer_cache.h | 42 ++- src/video_core/buffer_cache/buffer_cache_base.h | 19 +- src/video_core/buffer_cache/memory_tracker_base.h | 17 +- src/video_core/buffer_cache/word_manager.h | 384 +++++++++++----------- 6 files changed, 243 insertions(+), 322 deletions(-) (limited to 'src/video_core/buffer_cache/buffer_base.h') diff --git a/src/tests/video_core/memory_tracker.cpp b/src/tests/video_core/memory_tracker.cpp index 77d391f15e..3981907a22 100644 --- a/src/tests/video_core/memory_tracker.cpp +++ b/src/tests/video_core/memory_tracker.cpp @@ -427,7 +427,7 @@ TEST_CASE("MemoryTracker: Single page in large region", "[video_core]") { memory_track->MarkRegionAsCpuModified(c + WORD * 12 + PAGE * 8, PAGE); REQUIRE(memory_track->IsRegionCpuModified(c, WORD * 16)); - REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 10, WORD * 2)); + REQUIRE(!memory_track->IsRegionCpuModified(c + WORD * 10, WORD * 2)); REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 11, WORD * 2)); REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 12, WORD * 2)); REQUIRE(memory_track->IsRegionCpuModified(c + WORD * 12 + PAGE * 4, PAGE * 8)); @@ -535,6 +535,8 @@ TEST_CASE("MemoryTracker: Cached write downloads") { memory_track->MarkRegionAsGpuModified(c + PAGE, PAGE); int num = 0; memory_track->ForEachDownloadRangeAndClear(c, WORD, [&](u64 offset, u64 size) { ++num; }); + REQUIRE(num == 1); + num = 0; memory_track->ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) { ++num; }); REQUIRE(num == 0); REQUIRE(!memory_track->IsRegionCpuModified(c + PAGE, PAGE)); diff --git a/src/video_core/buffer_cache/buffer_base.h b/src/video_core/buffer_cache/buffer_base.h index 095f793878..9cbd95c4b0 100644 --- a/src/video_core/buffer_cache/buffer_base.h +++ b/src/video_core/buffer_cache/buffer_base.h @@ -38,10 +38,8 @@ public: static constexpr u64 BASE_PAGE_BITS = 16; static constexpr u64 BASE_PAGE_SIZE = 1ULL << BASE_PAGE_BITS; - explicit BufferBase(RasterizerInterface& rasterizer_, VAddr cpu_addr_, u64 size_bytes) - : cpu_addr{Common::AlignDown(cpu_addr_, BASE_PAGE_SIZE)}, - word_manager(cpu_addr, rasterizer_, - Common::AlignUp(size_bytes + (cpu_addr_ - cpu_addr), BASE_PAGE_SIZE)) {} + explicit BufferBase(RasterizerInterface& rasterizer_, VAddr cpu_addr_, u64 size_bytes_) + : cpu_addr{cpu_addr_}, size_bytes{size_bytes_} {} explicit BufferBase(NullBufferParams) {} @@ -51,88 +49,6 @@ public: BufferBase& operator=(BufferBase&&) = default; BufferBase(BufferBase&&) = default; - /// Returns the inclusive CPU modified range in a begin end pair - [[nodiscard]] std::pair ModifiedCpuRegion(VAddr query_cpu_addr, - u64 query_size) const noexcept { - const u64 offset = query_cpu_addr - cpu_addr; - return word_manager.template ModifiedRegion(offset, query_size); - } - - /// Returns the inclusive GPU modified range in a begin end pair - [[nodiscard]] std::pair ModifiedGpuRegion(VAddr query_cpu_addr, - u64 query_size) const noexcept { - const u64 offset = query_cpu_addr - cpu_addr; - return word_manager.template ModifiedRegion(offset, query_size); - } - - /// Returns true if a region has been modified from the CPU - [[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) const noexcept { - const u64 offset = query_cpu_addr - cpu_addr; - return word_manager.template IsRegionModified(offset, query_size); - } - - /// Returns true if a region has been modified from the GPU - [[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) const noexcept { - const u64 offset = query_cpu_addr - cpu_addr; - return word_manager.template IsRegionModified(offset, query_size); - } - - /// Mark region as CPU modified, notifying the rasterizer about this change - void MarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 size) { - word_manager.template ChangeRegionState(dirty_cpu_addr, size); - } - - /// Unmark region as CPU modified, notifying the rasterizer about this change - void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 size) { - word_manager.template ChangeRegionState(dirty_cpu_addr, size); - } - - /// Mark region as modified from the host GPU - void MarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 size) noexcept { - word_manager.template ChangeRegionState(dirty_cpu_addr, size); - } - - /// Unmark region as modified from the host GPU - void UnmarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 size) noexcept { - word_manager.template ChangeRegionState(dirty_cpu_addr, size); - } - - /// Mark region as modified from the CPU - /// but don't mark it as modified until FlusHCachedWrites is called. - void CachedCpuWrite(VAddr dirty_cpu_addr, u64 size) { - flags |= BufferFlagBits::CachedWrites; - word_manager.template ChangeRegionState(dirty_cpu_addr, size); - } - - /// Flushes cached CPU writes, and notify the rasterizer about the deltas - void FlushCachedWrites() noexcept { - flags &= ~BufferFlagBits::CachedWrites; - word_manager.FlushCachedWrites(); - } - - /// Call 'func' for each CPU modified range and unmark those pages as CPU modified - template - void ForEachUploadRange(VAddr query_cpu_range, u64 size, Func&& func) { - word_manager.template ForEachModifiedRange(query_cpu_range, size, true, func); - } - - /// Call 'func' for each GPU modified range and unmark those pages as GPU modified - template - void ForEachDownloadRange(VAddr query_cpu_range, u64 size, bool clear, Func&& func) { - word_manager.template ForEachModifiedRange(query_cpu_range, size, clear, func); - } - - template - void ForEachDownloadRangeAndClear(VAddr query_cpu_range, u64 size, Func&& func) { - word_manager.template ForEachModifiedRange(query_cpu_range, size, true, func); - } - - /// Call 'func' for each GPU modified range and unmark those pages as GPU modified - template - void ForEachDownloadRange(Func&& func) { - word_manager.template ForEachModifiedRange(cpu_addr, SizeBytes(), true, func); - } - /// Mark buffer as picked void Pick() noexcept { flags |= BufferFlagBits::Picked; @@ -179,11 +95,6 @@ public: return static_cast(other_cpu_addr - cpu_addr); } - /// Returns the size in bytes of the buffer - [[nodiscard]] u64 SizeBytes() const noexcept { - return word_manager.SizeBytes(); - } - size_t getLRUID() const noexcept { return lru_id; } @@ -192,12 +103,16 @@ public: lru_id = lru_id_; } + size_t SizeBytes() const { + return size_bytes; + } + private: VAddr cpu_addr = 0; - WordManager word_manager; BufferFlagBits flags{}; int stream_score = 0; size_t lru_id = SIZE_MAX; + size_t size_bytes = 0; }; } // namespace VideoCommon diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e5c626c364..7975564b52 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -21,6 +21,7 @@ BufferCache

::BufferCache(VideoCore::RasterizerInterface& rasterizer_, // Ensure the first slot is used for the null buffer void(slot_buffers.insert(runtime, NullBufferParams{})); common_ranges.clear(); + inline_buffer_id = NULL_BUFFER_ID; active_async_buffers = !Settings::IsGPULevelHigh(); @@ -442,9 +443,6 @@ template void BufferCache

::FlushCachedWrites() { cached_write_buffer_ids.clear(); memory_tracker.FlushCachedWrites(); - for (auto& interval : cached_ranges) { - ClearDownload(interval); - } cached_ranges.clear(); } @@ -659,8 +657,8 @@ bool BufferCache

::IsRegionGpuModified(VAddr addr, size_t size) { template bool BufferCache

::IsRegionRegistered(VAddr addr, size_t size) { const VAddr end_addr = addr + size; - const u64 page_end = Common::DivCeil(end_addr, PAGE_SIZE); - for (u64 page = addr >> PAGE_BITS; page < page_end;) { + const u64 page_end = Common::DivCeil(end_addr, CACHING_PAGESIZE); + for (u64 page = addr >> CACHING_PAGEBITS; page < page_end;) { const BufferId buffer_id = page_table[page]; if (!buffer_id) { ++page; @@ -672,7 +670,7 @@ bool BufferCache

::IsRegionRegistered(VAddr addr, size_t size) { if (buf_start_addr < end_addr && addr < buf_end_addr) { return true; } - page = Common::DivCeil(end_addr, PAGE_SIZE); + page = Common::DivCeil(end_addr, CACHING_PAGESIZE); } return false; } @@ -689,7 +687,7 @@ void BufferCache

::BindHostIndexBuffer() { const u32 offset = buffer.Offset(index_buffer.cpu_addr); const u32 size = index_buffer.size; const auto& draw_state = maxwell3d->draw_manager->GetDrawState(); - if (!draw_state.inline_index_draw_indexes.empty()) { + if (!draw_state.inline_index_draw_indexes.empty()) [[unlikely]] { if constexpr (USE_MEMORY_MAPS) { auto upload_staging = runtime.UploadStagingBuffer(size); std::array copies{ @@ -1001,12 +999,20 @@ void BufferCache

::UpdateIndexBuffer() { return; } flags[Dirty::IndexBuffer] = false; - if (!draw_state.inline_index_draw_indexes.empty()) { + if (!draw_state.inline_index_draw_indexes.empty()) [[unlikely]] { auto inline_index_size = static_cast(draw_state.inline_index_draw_indexes.size()); + u32 buffer_size = Common::AlignUp(inline_index_size, CACHING_PAGESIZE); + if (inline_buffer_id == NULL_BUFFER_ID) [[unlikely]] { + inline_buffer_id = CreateBuffer(0, buffer_size); + } + if (slot_buffers[inline_buffer_id].SizeBytes() < buffer_size) [[unlikely]] { + slot_buffers.erase(inline_buffer_id); + inline_buffer_id = CreateBuffer(0, buffer_size); + } index_buffer = Binding{ .cpu_addr = 0, .size = inline_index_size, - .buffer_id = FindBuffer(0, inline_index_size), + .buffer_id = inline_buffer_id, }; return; } @@ -1224,7 +1230,7 @@ BufferId BufferCache

::FindBuffer(VAddr cpu_addr, u32 size) { if (cpu_addr == 0) { return NULL_BUFFER_ID; } - const u64 page = cpu_addr >> PAGE_BITS; + const u64 page = cpu_addr >> CACHING_PAGEBITS; const BufferId buffer_id = page_table[page]; if (!buffer_id) { return CreateBuffer(cpu_addr, size); @@ -1253,8 +1259,9 @@ typename BufferCache

::OverlapResult BufferCache

::ResolveOverlaps(VAddr cpu .has_stream_leap = has_stream_leap, }; } - for (; cpu_addr >> PAGE_BITS < Common::DivCeil(end, PAGE_SIZE); cpu_addr += PAGE_SIZE) { - const BufferId overlap_id = page_table[cpu_addr >> PAGE_BITS]; + for (; cpu_addr >> CACHING_PAGEBITS < Common::DivCeil(end, CACHING_PAGESIZE); + cpu_addr += CACHING_PAGESIZE) { + const BufferId overlap_id = page_table[cpu_addr >> CACHING_PAGEBITS]; if (!overlap_id) { continue; } @@ -1280,11 +1287,11 @@ typename BufferCache

::OverlapResult BufferCache

::ResolveOverlaps(VAddr cpu // as a stream buffer. Increase the size to skip constantly recreating buffers. has_stream_leap = true; if (expands_right) { - begin -= PAGE_SIZE * 256; + begin -= CACHING_PAGESIZE * 256; cpu_addr = begin; } if (expands_left) { - end += PAGE_SIZE * 256; + end += CACHING_PAGESIZE * 256; } } } @@ -1317,6 +1324,9 @@ void BufferCache

::JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, template BufferId BufferCache

::CreateBuffer(VAddr cpu_addr, u32 wanted_size) { + VAddr cpu_addr_end = Common::AlignUp(cpu_addr + wanted_size, CACHING_PAGESIZE); + cpu_addr = Common::AlignDown(cpu_addr, CACHING_PAGESIZE); + wanted_size = static_cast(cpu_addr_end - cpu_addr); const OverlapResult overlap = ResolveOverlaps(cpu_addr, wanted_size); const u32 size = static_cast(overlap.end - overlap.begin); const BufferId new_buffer_id = slot_buffers.insert(runtime, rasterizer, overlap.begin, size); @@ -1354,8 +1364,8 @@ void BufferCache

::ChangeRegister(BufferId buffer_id) { } const VAddr cpu_addr_begin = buffer.CpuAddr(); const VAddr cpu_addr_end = cpu_addr_begin + size; - const u64 page_begin = cpu_addr_begin / PAGE_SIZE; - const u64 page_end = Common::DivCeil(cpu_addr_end, PAGE_SIZE); + const u64 page_begin = cpu_addr_begin / CACHING_PAGESIZE; + const u64 page_end = Common::DivCeil(cpu_addr_end, CACHING_PAGESIZE); for (u64 page = page_begin; page != page_end; ++page) { if constexpr (insert) { page_table[page] = buffer_id; diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 75cb98ba3e..656baa550b 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -90,10 +90,8 @@ template class BufferCache : public VideoCommon::ChannelSetupCaches { // Page size for caching purposes. // This is unrelated to the CPU page size and it can be changed as it seems optimal. - static constexpr u32 PAGE_BITS = 16; - static constexpr u64 PAGE_SIZE = u64{1} << PAGE_BITS; - static constexpr u32 CPU_PAGE_BITS = 12; - static constexpr u64 CPU_PAGE_SIZE = u64{1} << CPU_PAGE_BITS; + static constexpr u32 CACHING_PAGEBITS = 16; + static constexpr u64 CACHING_PAGESIZE = u64{1} << CACHING_PAGEBITS; static constexpr bool IS_OPENGL = P::IS_OPENGL; static constexpr bool HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS = @@ -112,6 +110,10 @@ class BufferCache : public VideoCommon::ChannelSetupCaches void ForEachBufferInRange(VAddr cpu_addr, u64 size, Func&& func) { - const u64 page_end = Common::DivCeil(cpu_addr + size, PAGE_SIZE); - for (u64 page = cpu_addr >> PAGE_BITS; page < page_end;) { + const u64 page_end = Common::DivCeil(cpu_addr + size, CACHING_PAGESIZE); + for (u64 page = cpu_addr >> CACHING_PAGEBITS; page < page_end;) { const BufferId buffer_id = page_table[page]; if (!buffer_id) { ++page; @@ -297,7 +299,7 @@ private: func(buffer_id, buffer); const VAddr end_addr = buffer.CpuAddr() + buffer.SizeBytes(); - page = Common::DivCeil(end_addr, PAGE_SIZE); + page = Common::DivCeil(end_addr, CACHING_PAGESIZE); } } @@ -568,10 +570,11 @@ private: u64 total_used_memory = 0; u64 minimum_memory = 0; u64 critical_memory = 0; + BufferId inline_buffer_id; bool active_async_buffers = false; - std::array> PAGE_BITS)> page_table; + std::array> CACHING_PAGEBITS)> page_table; }; } // namespace VideoCommon diff --git a/src/video_core/buffer_cache/memory_tracker_base.h b/src/video_core/buffer_cache/memory_tracker_base.h index 016d8430fc..4bc59017f1 100644 --- a/src/video_core/buffer_cache/memory_tracker_base.h +++ b/src/video_core/buffer_cache/memory_tracker_base.h @@ -132,8 +132,8 @@ public: void ForEachUploadRange(VAddr query_cpu_range, u64 query_size, Func&& func) { IteratePages(query_cpu_range, query_size, [&func](Manager* manager, u64 offset, size_t size) { - manager->template ForEachModifiedRange( - manager->GetCpuAddr() + offset, size, true, func); + manager->template ForEachModifiedRange( + manager->GetCpuAddr() + offset, size, func); }); } @@ -142,8 +142,13 @@ public: void ForEachDownloadRange(VAddr query_cpu_range, u64 query_size, bool clear, Func&& func) { IteratePages(query_cpu_range, query_size, [&func, clear](Manager* manager, u64 offset, size_t size) { - manager->template ForEachModifiedRange( - manager->GetCpuAddr() + offset, size, clear, func); + if (clear) { + manager->template ForEachModifiedRange( + manager->GetCpuAddr() + offset, size, func); + } else { + manager->template ForEachModifiedRange( + manager->GetCpuAddr() + offset, size, func); + } }); } @@ -151,8 +156,8 @@ public: void ForEachDownloadRangeAndClear(VAddr query_cpu_range, u64 query_size, Func&& func) { IteratePages(query_cpu_range, query_size, [&func](Manager* manager, u64 offset, size_t size) { - manager->template ForEachModifiedRange( - manager->GetCpuAddr() + offset, size, true, func); + manager->template ForEachModifiedRange( + manager->GetCpuAddr() + offset, size, func); }); } diff --git a/src/video_core/buffer_cache/word_manager.h b/src/video_core/buffer_cache/word_manager.h index 21729752b7..a42455045e 100644 --- a/src/video_core/buffer_cache/word_manager.h +++ b/src/video_core/buffer_cache/word_manager.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "common/alignment.h" @@ -20,9 +21,16 @@ constexpr u64 PAGES_PER_WORD = 64; constexpr u64 BYTES_PER_PAGE = Core::Memory::YUZU_PAGESIZE; constexpr u64 BYTES_PER_WORD = PAGES_PER_WORD * BYTES_PER_PAGE; +enum class Type { + CPU, + GPU, + CachedCPU, + Untracked, +}; + /// Vector tracking modified pages tightly packed with small vector optimization template -union WordsArray { +struct WordsArray { /// Returns the pointer to the words state [[nodiscard]] const u64* Pointer(bool is_short) const noexcept { return is_short ? stack.data() : heap; @@ -41,13 +49,13 @@ template struct Words { explicit Words() = default; explicit Words(u64 size_bytes_) : size_bytes{size_bytes_} { + num_words = Common::DivCeil(size_bytes, BYTES_PER_WORD); if (IsShort()) { cpu.stack.fill(~u64{0}); gpu.stack.fill(0); cached_cpu.stack.fill(0); untracked.stack.fill(~u64{0}); } else { - const size_t num_words = NumWords(); // Share allocation between CPU and GPU pages and set their default values u64* const alloc = new u64[num_words * 4]; cpu.heap = alloc; @@ -75,6 +83,7 @@ struct Words { Words& operator=(Words&& rhs) noexcept { Release(); size_bytes = rhs.size_bytes; + num_words = rhs.num_words; cpu = rhs.cpu; gpu = rhs.gpu; cached_cpu = rhs.cached_cpu; @@ -84,7 +93,7 @@ struct Words { } Words(Words&& rhs) noexcept - : size_bytes{rhs.size_bytes}, cpu{rhs.cpu}, gpu{rhs.gpu}, + : size_bytes{rhs.size_bytes}, num_words{rhs.num_words}, cpu{rhs.cpu}, gpu{rhs.gpu}, cached_cpu{rhs.cached_cpu}, untracked{rhs.untracked} { rhs.cpu.heap = nullptr; } @@ -94,12 +103,12 @@ struct Words { /// Returns true when the buffer fits in the small vector optimization [[nodiscard]] bool IsShort() const noexcept { - return size_bytes <= stack_words * BYTES_PER_WORD; + return num_words <= stack_words; } /// Returns the number of words of the buffer [[nodiscard]] size_t NumWords() const noexcept { - return Common::DivCeil(size_bytes, BYTES_PER_WORD); + return num_words; } /// Release buffer resources @@ -110,20 +119,40 @@ struct Words { } } + template + std::span Span() noexcept { + if constexpr (type == Type::CPU) { + return std::span(cpu.Pointer(IsShort()), num_words); + } else if constexpr (type == Type::GPU) { + return std::span(gpu.Pointer(IsShort()), num_words); + } else if constexpr (type == Type::CachedCPU) { + return std::span(cached_cpu.Pointer(IsShort()), num_words); + } else if constexpr (type == Type::Untracked) { + return std::span(untracked.Pointer(IsShort()), num_words); + } + } + + template + std::span Span() const noexcept { + if constexpr (type == Type::CPU) { + return std::span(cpu.Pointer(IsShort()), num_words); + } else if constexpr (type == Type::GPU) { + return std::span(gpu.Pointer(IsShort()), num_words); + } else if constexpr (type == Type::CachedCPU) { + return std::span(cached_cpu.Pointer(IsShort()), num_words); + } else if constexpr (type == Type::Untracked) { + return std::span(untracked.Pointer(IsShort()), num_words); + } + } + u64 size_bytes = 0; + size_t num_words = 0; WordsArray cpu; WordsArray gpu; WordsArray cached_cpu; WordsArray untracked; }; -enum class Type { - CPU, - GPU, - CachedCPU, - Untracked, -}; - template class WordManager { public: @@ -140,6 +169,69 @@ public: return cpu_addr; } + static u64 ExtractBits(u64 word, size_t page_start, size_t page_end) { + constexpr size_t number_bits = sizeof(u64) * 8; + const size_t limit_page_end = number_bits - std::min(page_end, number_bits); + u64 bits = (word >> page_start) << page_start; + bits = (bits << limit_page_end) >> limit_page_end; + return bits; + } + + static std::pair GetWordPage(VAddr address) { + const size_t converted_address = static_cast(address); + const size_t word_number = converted_address / BYTES_PER_WORD; + const size_t amount_pages = converted_address % BYTES_PER_WORD; + return std::make_pair(word_number, amount_pages / BYTES_PER_PAGE); + } + + template + void IterateWords(size_t offset, size_t size, Func&& func) const { + using FuncReturn = std::invoke_result_t; + static constexpr bool BOOL_BREAK = std::is_same_v; + const size_t start = static_cast(std::max(static_cast(offset), 0LL)); + const size_t end = static_cast(std::max(static_cast(offset + size), 0LL)); + if (start >= SizeBytes() || end <= start) { + return; + } + auto [start_word, start_page] = GetWordPage(start); + auto [end_word, end_page] = GetWordPage(end + BYTES_PER_PAGE - 1ULL); + const size_t num_words = NumWords(); + start_word = std::min(start_word, num_words); + end_word = std::min(end_word, num_words); + const size_t diff = end_word - start_word; + end_word += (end_page + PAGES_PER_WORD - 1ULL) / PAGES_PER_WORD; + end_word = std::min(end_word, num_words); + end_page += diff * PAGES_PER_WORD; + constexpr u64 base_mask{~0ULL}; + for (size_t word_index = start_word; word_index < end_word; word_index++) { + const u64 mask = ExtractBits(base_mask, start_page, end_page); + start_page = 0; + end_page -= PAGES_PER_WORD; + if constexpr (BOOL_BREAK) { + if (func(word_index, mask)) { + return; + } + } else { + func(word_index, mask); + } + } + } + + template + void IteratePages(u64 mask, Func&& func) const { + size_t offset = 0; + while (mask != 0) { + const size_t empty_bits = std::countr_zero(mask); + offset += empty_bits; + mask = mask >> empty_bits; + + const size_t continuous_bits = std::countr_one(mask); + func(offset, continuous_bits); + mask = continuous_bits < PAGES_PER_WORD ? (mask >> continuous_bits) : 0; + offset += continuous_bits; + } + } + /** * Change the state of a range of pages * @@ -147,47 +239,33 @@ public: * @param size Size in bytes to mark or unmark as modified */ template - void ChangeRegionState(u64 dirty_addr, s64 size) noexcept(type == Type::GPU) { - const s64 difference = dirty_addr - cpu_addr; - const u64 offset = std::max(difference, 0); - size += std::min(difference, 0); - if (offset >= SizeBytes() || size < 0) { - return; - } - u64* const untracked_words = Array(); - u64* const state_words = Array(); - const u64 offset_end = std::min(offset + size, SizeBytes()); - const u64 begin_page_index = offset / BYTES_PER_PAGE; - const u64 begin_word_index = begin_page_index / PAGES_PER_WORD; - const u64 end_page_index = Common::DivCeil(offset_end, BYTES_PER_PAGE); - const u64 end_word_index = Common::DivCeil(end_page_index, PAGES_PER_WORD); - u64 page_index = begin_page_index % PAGES_PER_WORD; - u64 word_index = begin_word_index; - while (word_index < end_word_index) { - const u64 next_word_first_page = (word_index + 1) * PAGES_PER_WORD; - const u64 left_offset = - std::min(next_word_first_page - end_page_index, PAGES_PER_WORD) % PAGES_PER_WORD; - const u64 right_offset = page_index; - u64 bits = ~u64{0}; - bits = (bits >> right_offset) << right_offset; - bits = (bits << left_offset) >> left_offset; + void ChangeRegionState(u64 dirty_addr, u64 size) noexcept(type == Type::GPU) { + std::span state_words = words.template Span(); + [[maybe_unused]] std::span untracked_words = words.template Span(); + [[maybe_unused]] std::span cached_words = words.template Span(); + IterateWords(dirty_addr - cpu_addr, size, [&](size_t index, u64 mask) { if constexpr (type == Type::CPU || type == Type::CachedCPU) { - NotifyRasterizer(word_index, untracked_words[word_index], bits); + NotifyRasterizer(index, untracked_words[index], mask); } if constexpr (enable) { - state_words[word_index] |= bits; + state_words[index] |= mask; if constexpr (type == Type::CPU || type == Type::CachedCPU) { - untracked_words[word_index] |= bits; + untracked_words[index] |= mask; + } + if constexpr (type == Type::CPU) { + cached_words[index] &= ~mask; } } else { - state_words[word_index] &= ~bits; + if constexpr (type == Type::CPU) { + const u64 word = state_words[index] & mask; + cached_words[index] &= ~word; + } + state_words[index] &= ~mask; if constexpr (type == Type::CPU || type == Type::CachedCPU) { - untracked_words[word_index] &= ~bits; + untracked_words[index] &= ~mask; } } - page_index = 0; - ++word_index; - } + }); } /** @@ -198,119 +276,59 @@ public: * @param size Size in bytes of the CPU range to loop over * @param func Function to call for each turned off region */ - template - void ForEachModifiedRange(VAddr query_cpu_range, s64 size, bool clear, Func&& func) { + template + void ForEachModifiedRange(VAddr query_cpu_range, s64 size, Func&& func) { static_assert(type != Type::Untracked); - const s64 difference = query_cpu_range - cpu_addr; - const u64 query_begin = std::max(difference, 0); - size += std::min(difference, 0); - if (query_begin >= SizeBytes() || size < 0) { - return; - } - [[maybe_unused]] u64* const untracked_words = Array(); - [[maybe_unused]] u64* const cpu_words = Array(); - u64* const state_words = Array(); - const u64 query_end = query_begin + std::min(static_cast(size), SizeBytes()); - u64* const words_begin = state_words + query_begin / BYTES_PER_WORD; - u64* const words_end = state_words + Common::DivCeil(query_end, BYTES_PER_WORD); - u64 first_page = (query_begin / BYTES_PER_PAGE) % PAGES_PER_WORD; - - const auto modified = [](u64 word) { return word != 0; }; - const auto first_modified_word = std::find_if(words_begin, words_end, modified); - if (first_modified_word == words_end) { - // Exit early when the buffer is not modified - return; - } - if (first_modified_word != words_begin) { - first_page = 0; - } - std::reverse_iterator first_word_reverse(first_modified_word); - std::reverse_iterator last_word_iterator(words_end); - auto last_word_result = std::find_if(last_word_iterator, first_word_reverse, modified); - u64* const last_modified_word = &(*last_word_result) + 1; - - const u64 word_index_begin = std::distance(state_words, first_modified_word); - const u64 word_index_end = std::distance(state_words, last_modified_word); - const unsigned local_page_begin = std::countr_zero(*first_modified_word); - const unsigned local_page_end = - static_cast(PAGES_PER_WORD) - std::countl_zero(last_modified_word[-1]); - const u64 word_page_begin = word_index_begin * PAGES_PER_WORD; - const u64 word_page_end = (word_index_end - 1) * PAGES_PER_WORD; - const u64 query_page_begin = query_begin / BYTES_PER_PAGE; - const u64 query_page_end = Common::DivCeil(query_end, BYTES_PER_PAGE); - const u64 page_index_begin = std::max(word_page_begin + local_page_begin, query_page_begin); - const u64 page_index_end = std::min(word_page_end + local_page_end, query_page_end); - const u64 first_word_page_begin = page_index_begin % PAGES_PER_WORD; - const u64 last_word_page_end = (page_index_end - 1) % PAGES_PER_WORD + 1; - - u64 page_begin = std::max(first_word_page_begin, first_page); - u64 current_base = 0; - u64 current_size = 0; - bool on_going = false; - for (u64 word_index = word_index_begin; word_index < word_index_end; ++word_index) { - const bool is_last_word = word_index + 1 == word_index_end; - const u64 page_end = is_last_word ? last_word_page_end : PAGES_PER_WORD; - const u64 right_offset = page_begin; - const u64 left_offset = PAGES_PER_WORD - page_end; - u64 bits = ~u64{0}; - bits = (bits >> right_offset) << right_offset; - bits = (bits << left_offset) >> left_offset; - - const u64 current_word = state_words[word_index] & bits; - if (clear) { - state_words[word_index] &= ~bits; - } - - if constexpr (type == Type::CachedCPU) { - NotifyRasterizer(word_index, untracked_words[word_index], current_word); - untracked_words[word_index] |= current_word; - cpu_words[word_index] |= current_word; - } - - if constexpr (type == Type::CPU) { - const u64 current_bits = untracked_words[word_index] & bits; - untracked_words[word_index] &= ~bits; - NotifyRasterizer(word_index, current_bits, ~u64{0}); - } - const u64 word = current_word & ~(type == Type::GPU ? untracked_words[word_index] : 0); - u64 page = page_begin; - page_begin = 0; - - while (page < page_end) { - const int empty_bits = std::countr_zero(word >> page); - if (on_going && empty_bits != 0) { - InvokeModifiedRange(func, current_size, current_base); - current_size = 0; - on_going = false; + std::span state_words = words.template Span(); + [[maybe_unused]] std::span untracked_words = words.template Span(); + [[maybe_unused]] std::span cached_words = words.template Span(); + const size_t offset = query_cpu_range - cpu_addr; + bool pending = false; + size_t pending_offset{}; + size_t pending_pointer{}; + const auto release = [&]() { + func(cpu_addr + pending_offset * BYTES_PER_PAGE, + (pending_pointer - pending_offset) * BYTES_PER_PAGE); + }; + IterateWords(offset, size, [&](size_t index, u64 mask) { + const u64 word = state_words[index] & mask; + if constexpr (clear) { + if constexpr (type == Type::CPU || type == Type::CachedCPU) { + NotifyRasterizer(index, untracked_words[index], mask); } - if (empty_bits == PAGES_PER_WORD) { - break; + state_words[index] &= ~mask; + if constexpr (type == Type::CPU || type == Type::CachedCPU) { + untracked_words[index] &= ~mask; } - page += empty_bits; - - const int continuous_bits = std::countr_one(word >> page); - if (!on_going && continuous_bits != 0) { - current_base = word_index * PAGES_PER_WORD + page; - on_going = true; + if constexpr (type == Type::CPU) { + cached_words[index] &= ~word; } - current_size += continuous_bits; - page += continuous_bits; } - } - if (on_going && current_size > 0) { - InvokeModifiedRange(func, current_size, current_base); + const size_t base_offset = index * PAGES_PER_WORD; + IteratePages(word, [&](size_t pages_offset, size_t pages_size) { + const auto reset = [&]() { + pending_offset = base_offset + pages_offset; + pending_pointer = base_offset + pages_offset + pages_size; + }; + if (!pending) { + reset(); + pending = true; + return; + } + if (pending_pointer == base_offset + pages_offset) { + pending_pointer += pages_size; + return; + } + release(); + reset(); + }); + }); + if (pending) { + release(); } } - template - void InvokeModifiedRange(Func&& func, u64 current_size, u64 current_base) { - const u64 current_size_bytes = current_size * BYTES_PER_PAGE; - const u64 offset_begin = current_base * BYTES_PER_PAGE; - const u64 offset_end = std::min(offset_begin + current_size_bytes, SizeBytes()); - func(cpu_addr + offset_begin, offset_end - offset_begin); - } - /** * Returns true when a region has been modified * @@ -321,27 +339,17 @@ public: [[nodiscard]] bool IsRegionModified(u64 offset, u64 size) const noexcept { static_assert(type != Type::Untracked); - const u64* const untracked_words = Array(); - const u64* const state_words = Array(); - const u64 num_query_words = size / BYTES_PER_WORD + 1; - const u64 word_begin = offset / BYTES_PER_WORD; - const u64 word_end = std::min(word_begin + num_query_words, NumWords()); - const u64 page_limit = Common::DivCeil(offset + size, BYTES_PER_PAGE); - u64 page_index = (offset / BYTES_PER_PAGE) % PAGES_PER_WORD; - for (u64 word_index = word_begin; word_index < word_end; ++word_index, page_index = 0) { - const u64 off_word = type == Type::GPU ? untracked_words[word_index] : 0; - const u64 word = state_words[word_index] & ~off_word; - if (word == 0) { - continue; - } - const u64 page_end = std::min((word_index + 1) * PAGES_PER_WORD, page_limit); - const u64 local_page_end = page_end % PAGES_PER_WORD; - const u64 page_end_shift = (PAGES_PER_WORD - local_page_end) % PAGES_PER_WORD; - if (((word >> page_index) << page_index) << page_end_shift != 0) { + const std::span state_words = words.template Span(); + bool result = false; + IterateWords(offset, size, [&](size_t index, u64 mask) { + const u64 word = state_words[index] & mask; + if (word != 0) { + result = true; return true; } - } - return false; + return false; + }); + return result; } /** @@ -353,34 +361,20 @@ public: template [[nodiscard]] std::pair ModifiedRegion(u64 offset, u64 size) const noexcept { static_assert(type != Type::Untracked); - const u64* const state_words = Array(); - const u64 num_query_words = size / BYTES_PER_WORD + 1; - const u64 word_begin = offset / BYTES_PER_WORD; - const u64 word_end = std::min(word_begin + num_query_words, NumWords()); - const u64 page_base = offset / BYTES_PER_PAGE; - u64 page_begin = page_base & (PAGES_PER_WORD - 1); - u64 page_end = - Common::DivCeil(offset + size, BYTES_PER_PAGE) - (page_base & ~(PAGES_PER_WORD - 1)); + const std::span state_words = words.template Span(); u64 begin = std::numeric_limits::max(); u64 end = 0; - for (u64 word_index = word_begin; word_index < word_end; ++word_index) { - const u64 base_mask = (1ULL << page_begin) - 1ULL; - const u64 end_mask = page_end >= PAGES_PER_WORD ? 0ULL : ~((1ULL << page_end) - 1ULL); - const u64 off_word = end_mask | base_mask; - const u64 word = state_words[word_index] & ~off_word; + IterateWords(offset, size, [&](size_t index, u64 mask) { + const u64 word = state_words[index] & mask; if (word == 0) { - page_begin = 0; - page_end -= PAGES_PER_WORD; - continue; + return; } const u64 local_page_begin = std::countr_zero(word); const u64 local_page_end = PAGES_PER_WORD - std::countl_zero(word); - const u64 page_index = word_index * PAGES_PER_WORD; + const u64 page_index = index * PAGES_PER_WORD; begin = std::min(begin, page_index + local_page_begin); end = page_index + local_page_end; - page_begin = 0; - page_end -= PAGES_PER_WORD; - } + }); static constexpr std::pair EMPTY{0, 0}; return begin < end ? std::make_pair(begin * BYTES_PER_PAGE, end * BYTES_PER_PAGE) : EMPTY; } @@ -454,18 +448,10 @@ private: void NotifyRasterizer(u64 word_index, u64 current_bits, u64 new_bits) const { u64 changed_bits = (add_to_rasterizer ? current_bits : ~current_bits) & new_bits; VAddr addr = cpu_addr + word_index * BYTES_PER_WORD; - while (changed_bits != 0) { - const int empty_bits = std::countr_zero(changed_bits); - addr += empty_bits * BYTES_PER_PAGE; - changed_bits >>= empty_bits; - - const u32 continuous_bits = std::countr_one(changed_bits); - const u64 size = continuous_bits * BYTES_PER_PAGE; - const VAddr begin_addr = addr; - addr += size; - changed_bits = continuous_bits < PAGES_PER_WORD ? (changed_bits >> continuous_bits) : 0; - rasterizer->UpdatePagesCachedCount(begin_addr, size, add_to_rasterizer ? 1 : -1); - } + IteratePages(changed_bits, [&](size_t offset, size_t size) { + rasterizer->UpdatePagesCachedCount(addr + offset * BYTES_PER_PAGE, + size * BYTES_PER_PAGE, add_to_rasterizer ? 1 : -1); + }); } VAddr cpu_addr = 0; -- cgit v1.2.3