From 9db159da71875db9d9b9fbb7697fc2c2f9575cf2 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Mon, 25 Dec 2023 07:32:16 +0100 Subject: SMMU: Initial adaptation to video_core. --- src/core/memory.cpp | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) (limited to 'src/core/memory.cpp') diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 8176a41be8..609e775aed 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -24,6 +24,8 @@ #include "core/hle/kernel/k_process.h" #include "core/memory.h" #include "video_core/gpu.h" +#include "video_core/host1x/gpu_device_memory_manager.h" +#include "video_core/host1x/host1x.h" #include "video_core/rasterizer_download_area.h" namespace Core::Memory { @@ -638,15 +640,16 @@ struct Memory::Impl { base * YUZU_PAGESIZE, (base + size) * YUZU_PAGESIZE); // During boot, current_page_table might not be set yet, in which case we need not flush - if (system.IsPoweredOn()) { + /*if (system.IsPoweredOn()) { auto& gpu = system.GPU(); for (u64 i = 0; i < size; i++) { const auto page = base + i; if (page_table.pointers[page].Type() == Common::PageType::RasterizerCachedMemory) { + gpu.FlushAndInvalidateRegion(page << YUZU_PAGEBITS, YUZU_PAGESIZE); } } - } + }*/ const auto end = base + size; ASSERT_MSG(end <= page_table.pointers.size(), "out of range mapping at {:016X}", @@ -811,10 +814,15 @@ struct Memory::Impl { return true; } - void HandleRasterizerDownload(VAddr address, size_t size) { + void HandleRasterizerDownload(VAddr v_address, size_t size) { + const auto* p = GetPointerImpl( + v_address, []() {}, []() {}); + auto& gpu_device_memory = system.Host1x().MemoryManager(); + DAddr address = + gpu_device_memory.GetAddressFromPAddr(system.DeviceMemory().GetRawPhysicalAddr(p)); const size_t core = system.GetCurrentHostThreadID(); auto& current_area = rasterizer_read_areas[core]; - const VAddr end_address = address + size; + const DAddr end_address = address + size; if (current_area.start_address <= address && end_address <= current_area.end_address) [[likely]] { return; @@ -822,7 +830,10 @@ struct Memory::Impl { current_area = system.GPU().OnCPURead(address, size); } - void HandleRasterizerWrite(VAddr address, size_t size) { + void HandleRasterizerWrite(VAddr v_address, size_t size) { + const auto* p = GetPointerImpl( + v_address, []() {}, []() {}); + PAddr address = system.DeviceMemory().GetRawPhysicalAddr(p); constexpr size_t sys_core = Core::Hardware::NUM_CPU_CORES - 1; const size_t core = std::min(system.GetCurrentHostThreadID(), sys_core); // any other calls threads go to syscore. @@ -836,7 +847,7 @@ struct Memory::Impl { } }); auto& current_area = rasterizer_write_areas[core]; - VAddr subaddress = address >> YUZU_PAGEBITS; + PAddr subaddress = address >> YUZU_PAGEBITS; bool do_collection = current_area.last_address == subaddress; if (!do_collection) [[unlikely]] { do_collection = system.GPU().OnCPUWrite(address, size); @@ -849,7 +860,7 @@ struct Memory::Impl { } struct GPUDirtyState { - VAddr last_address; + PAddr last_address; }; void InvalidateRegion(Common::ProcessAddress dest_addr, size_t size) { -- cgit v1.2.3 From f11218a03e6646b593155243bf0e5accb04ef480 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Fri, 29 Dec 2023 07:53:52 +0100 Subject: SMMU: Implement physical memory mirroring --- src/core/device_memory_manager.h | 27 +++- src/core/device_memory_manager.inc | 154 ++++++++++++++++++++++- src/core/hle/service/nvdrv/core/container.cpp | 6 +- src/core/hle/service/nvdrv/core/nvmap.cpp | 7 +- src/core/hle/service/nvdrv/core/nvmap.h | 6 +- src/core/memory.cpp | 53 ++++---- src/video_core/renderer_opengl/gl_rasterizer.cpp | 7 +- src/video_core/renderer_vulkan/vk_rasterizer.cpp | 6 +- 8 files changed, 226 insertions(+), 40 deletions(-) (limited to 'src/core/memory.cpp') diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index 1a63cbd093..7c7726348e 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -10,8 +10,10 @@ #include #include "common/common_types.h" +#include "common/scratch_buffer.h" #include "common/virtual_buffer.h" + namespace Core { class DeviceMemory; @@ -49,9 +51,25 @@ public: template const T* GetPointer(DAddr address) const; - DAddr GetAddressFromPAddr(PAddr address) const { + template + void ApplyOpOnPAddr(PAddr address, Common::ScratchBuffer& buffer, Func&& operation) { DAddr subbits = static_cast(address & page_mask); - return (static_cast(compressed_device_addr[(address >> page_bits)]) << page_bits) + subbits; + const u32 base = compressed_device_addr[(address >> page_bits)]; + if ((base >> MULTI_FLAG_BITS) == 0) [[likely]] { + const DAddr d_address = static_cast(base << page_bits) + subbits; + operation(d_address); + return; + } + InnerGatherDeviceAddresses(buffer, address); + for (u32 value : buffer) { + operation(static_cast(value << page_bits) + subbits); + } + } + + template + void ApplyOpOnPointer(const u8* p, Common::ScratchBuffer& buffer, Func&& operation) { + PAddr address = GetRawPhysicalAddr(p); + ApplyOpOnPAddr(address, buffer, operation); } PAddr GetPhysicalRawAddressFromDAddr(DAddr address) const { @@ -98,6 +116,9 @@ private: static constexpr size_t page_size = 1ULL << page_bits; static constexpr size_t page_mask = page_size - 1ULL; static constexpr u32 physical_address_base = 1U << page_bits; + static constexpr u32 MULTI_FLAG_BITS = 31; + static constexpr u32 MULTI_FLAG = 1U << MULTI_FLAG_BITS; + static constexpr u32 MULTI_MASK = ~MULTI_FLAG; template T* GetPointerFromRaw(PAddr addr) { @@ -117,6 +138,8 @@ private: void WalkBlock(const DAddr addr, const std::size_t size, auto on_unmapped, auto on_memory, auto increment); + void InnerGatherDeviceAddresses(Common::ScratchBuffer& buffer, PAddr address); + std::unique_ptr> impl; const uintptr_t physical_base; diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 8c5f82d315..4fb3ad3ab5 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -18,10 +18,117 @@ namespace Core { +namespace { + +class PhysicalAddressContainer { +public: + PhysicalAddressContainer() = default; + ~PhysicalAddressContainer() = default; + + void GatherValues(u32 start_entry, Common::ScratchBuffer& buffer) { + buffer.resize(8); + buffer.resize(0); + size_t index = 0; + const auto add_value = [&](u32 value) { + buffer[index] = value; + index++; + buffer.resize(index); + }; + + u32 iter_entry = start_entry; + Entry* current = &storage[iter_entry - 1]; + add_value(current->value); + while (current->next_entry != 0) { + iter_entry = current->next_entry; + current = &storage[iter_entry - 1]; + add_value(current->value); + } + } + + u32 Register(u32 value) { + return RegisterImplementation(value); + } + + void Register(u32 value, u32 start_entry) { + auto entry_id = RegisterImplementation(value); + u32 iter_entry = start_entry; + Entry* current = &storage[iter_entry - 1]; + while (current->next_entry != 0) { + iter_entry = current->next_entry; + current = &storage[iter_entry - 1]; + } + current->next_entry = entry_id; + } + + std::pair Unregister(u32 value, u32 start_entry) { + u32 iter_entry = start_entry; + Entry* previous{}; + Entry* current = &storage[iter_entry - 1]; + Entry* next{}; + bool more_than_one_remaining = false; + u32 result_start{start_entry}; + size_t count = 0; + while (current->value != value) { + count++; + previous = current; + iter_entry = current->next_entry; + current = &storage[iter_entry - 1]; + } + // Find next + u32 next_entry = current->next_entry; + if (next_entry != 0) { + next = &storage[next_entry - 1]; + more_than_one_remaining = next->next_entry != 0; + } + if (previous) { + previous->next_entry = next_entry; + } else { + result_start = next_entry; + } + free_entries.emplace_back(iter_entry); + return std::make_pair(more_than_one_remaining || count > 1, result_start); + } + + u32 ReleaseEntry(u32 start_entry) { + Entry* current = &storage[start_entry - 1]; + free_entries.emplace_back(start_entry); + return current->value; + } + +private: + u32 RegisterImplementation(u32 value) { + auto entry_id = GetNewEntry(); + auto& entry = storage[entry_id - 1]; + entry.next_entry = 0; + entry.value = value; + return entry_id; + } + u32 GetNewEntry() { + if (!free_entries.empty()) { + u32 result = free_entries.front(); + free_entries.pop_front(); + return result; + } + storage.emplace_back(); + u32 new_entry = static_cast(storage.size()); + return new_entry; + } + + struct Entry { + u32 next_entry{}; + u32 value{}; + }; + + std::deque storage; + std::deque free_entries; +}; + struct EmptyAllocator { EmptyAllocator([[maybe_unused]] DAddr address) {} }; +} // namespace + template struct DeviceMemoryManagerAllocator { static constexpr bool supports_pinning = DTraits::supports_pinning; @@ -38,6 +145,7 @@ struct DeviceMemoryManagerAllocator { std::conditional_t, EmptyAllocator> pin_allocator; Common::FlatAllocator main_allocator; + PhysicalAddressContainer multi_dev_address; /// Returns true when vaddr -> vaddr+size is fully contained in the buffer template @@ -109,6 +217,9 @@ DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memo cpu_backing_address(device_as_size >> Memory::YUZU_PAGEBITS) { impl = std::make_unique>(); cached_pages = std::make_unique(); + for (size_t i = 0; i < 1ULL << (33 - 12); i++) { + compressed_device_addr[i] = 0; + } } template @@ -155,8 +266,19 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size } auto phys_addr = static_cast(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U; compressed_physical_ptr[start_page_d + i] = phys_addr; - compressed_device_addr[phys_addr - 1U] = static_cast(start_page_d + i); InsertCPUBacking(start_page_d + i, new_vaddress, process_id); + const u32 base_dev = compressed_device_addr[phys_addr - 1U]; + const u32 new_dev = static_cast(start_page_d + i); + if (base_dev == 0) [[likely]] { + compressed_device_addr[phys_addr - 1U] = new_dev; + continue; + } + u32 start_id = base_dev & MULTI_MASK; + if ((base_dev >> MULTI_FLAG_BITS) == 0) { + start_id = impl->multi_dev_address.Register(base_dev); + compressed_device_addr[phys_addr - 1U] = MULTI_FLAG | start_id; + } + impl->multi_dev_address.Register(new_dev, start_id); } } @@ -170,12 +292,38 @@ void DeviceMemoryManager::Unmap(DAddr address, size_t size) { auto phys_addr = compressed_physical_ptr[start_page_d + i]; compressed_physical_ptr[start_page_d + i] = 0; cpu_backing_address[start_page_d + i] = 0; - if (phys_addr != 0) { - compressed_device_addr[phys_addr - 1] = 0; + if (phys_addr != 0) [[likely]] { + const u32 base_dev = compressed_device_addr[phys_addr - 1U]; + if ((base_dev >> MULTI_FLAG_BITS) == 0) [[likely]] { + compressed_device_addr[phys_addr - 1] = 0; + continue; + } + const auto [more_entries, new_start] = impl->multi_dev_address.Unregister( + static_cast(start_page_d + i), base_dev & MULTI_MASK); + if (!more_entries) { + compressed_device_addr[phys_addr - 1] = + impl->multi_dev_address.ReleaseEntry(new_start); + continue; + } + compressed_device_addr[phys_addr - 1] = new_start | MULTI_FLAG; } } } +template +void DeviceMemoryManager::InnerGatherDeviceAddresses(Common::ScratchBuffer& buffer, + PAddr address) { + size_t phys_addr = address >> page_bits; + std::scoped_lock lk(mapping_guard); + u32 backing = compressed_device_addr[phys_addr]; + if ((backing >> MULTI_FLAG_BITS) != 0) { + impl->multi_dev_address.GatherValues(backing & MULTI_MASK, buffer); + return; + } + buffer.resize(1); + buffer[0] = backing; +} + template template T* DeviceMemoryManager::GetPointer(DAddr address) { diff --git a/src/core/hle/service/nvdrv/core/container.cpp b/src/core/hle/service/nvdrv/core/container.cpp index 7c2231fe6e..e12ce05c13 100644 --- a/src/core/hle/service/nvdrv/core/container.cpp +++ b/src/core/hle/service/nvdrv/core/container.cpp @@ -16,8 +16,8 @@ namespace Service::Nvidia::NvCore { struct ContainerImpl { - explicit ContainerImpl(Tegra::Host1x::Host1x& host1x_) - : host1x{host1x_}, file{host1x_}, manager{host1x_}, device_file_data{} {} + explicit ContainerImpl(Container& core, Tegra::Host1x::Host1x& host1x_) + : host1x{host1x_}, file{core, host1x_}, manager{host1x_}, device_file_data{} {} Tegra::Host1x::Host1x& host1x; NvMap file; SyncpointManager manager; @@ -29,7 +29,7 @@ struct ContainerImpl { }; Container::Container(Tegra::Host1x::Host1x& host1x_) { - impl = std::make_unique(host1x_); + impl = std::make_unique(*this, host1x_); } Container::~Container() = default; diff --git a/src/core/hle/service/nvdrv/core/nvmap.cpp b/src/core/hle/service/nvdrv/core/nvmap.cpp index 7879c6f04a..e4168a37c5 100644 --- a/src/core/hle/service/nvdrv/core/nvmap.cpp +++ b/src/core/hle/service/nvdrv/core/nvmap.cpp @@ -7,6 +7,7 @@ #include "common/alignment.h" #include "common/assert.h" #include "common/logging/log.h" +#include "core/hle/service/nvdrv/core/container.h" #include "core/hle/service/nvdrv/core/nvmap.h" #include "core/memory.h" #include "video_core/host1x/host1x.h" @@ -64,7 +65,7 @@ NvResult NvMap::Handle::Duplicate(bool internal_session) { return NvResult::Success; } -NvMap::NvMap(Tegra::Host1x::Host1x& host1x_) : host1x{host1x_} {} +NvMap::NvMap(Container& core_, Tegra::Host1x::Host1x& host1x_) : host1x{host1x_}, core{core_} {} void NvMap::AddHandle(std::shared_ptr handle_description) { std::scoped_lock lock(handles_lock); @@ -160,6 +161,8 @@ DAddr NvMap::PinHandle(NvMap::Handle::Id handle, size_t session_id, bool low_are // If not then allocate some space and map it DAddr address{}; auto& smmu = host1x.MemoryManager(); + auto* session = core.GetSession(session_id); + auto allocate = std::bind(&Tegra::MaxwellDeviceMemoryManager::Allocate, &smmu, _1); //: std::bind(&Tegra::MaxwellDeviceMemoryManager::Allocate, &smmu, _1); while ((address = allocate(static_cast(handle_description->aligned_size))) == 0) { @@ -179,7 +182,7 @@ DAddr NvMap::PinHandle(NvMap::Handle::Id handle, size_t session_id, bool low_are handle_description->d_address = address; smmu.Map(address, handle_description->address, handle_description->aligned_size, - session_id); + session->smmu_id); } handle_description->pins++; diff --git a/src/core/hle/service/nvdrv/core/nvmap.h b/src/core/hle/service/nvdrv/core/nvmap.h index e9e9e8b5be..7dd6d26c3c 100644 --- a/src/core/hle/service/nvdrv/core/nvmap.h +++ b/src/core/hle/service/nvdrv/core/nvmap.h @@ -25,6 +25,8 @@ class Host1x; } // namespace Tegra namespace Service::Nvidia::NvCore { + +class Container; /** * @brief The nvmap core class holds the global state for nvmap and provides methods to manage * handles @@ -109,7 +111,7 @@ public: bool can_unlock; //!< If the address region is ready to be unlocked }; - explicit NvMap(Tegra::Host1x::Host1x& host1x); + explicit NvMap(Container& core, Tegra::Host1x::Host1x& host1x); /** * @brief Creates an unallocated handle of the given size @@ -173,5 +175,7 @@ private: * @return If the handle was removed from the map */ bool TryRemoveHandle(const Handle& handle_description); + + Container& core; }; } // namespace Service::Nvidia::NvCore diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 609e775aed..f126840cbf 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -44,7 +44,8 @@ bool AddressSpaceContains(const Common::PageTable& table, const Common::ProcessA // from outside classes. This also allows modification to the internals of the memory // subsystem without needing to rebuild all files that make use of the memory interface. struct Memory::Impl { - explicit Impl(Core::System& system_) : system{system_} {} + explicit Impl(Core::System& system_) + : system{system_} {} void SetCurrentPageTable(Kernel::KProcess& process) { current_page_table = &process.GetPageTable().GetImpl(); @@ -817,26 +818,31 @@ struct Memory::Impl { void HandleRasterizerDownload(VAddr v_address, size_t size) { const auto* p = GetPointerImpl( v_address, []() {}, []() {}); - auto& gpu_device_memory = system.Host1x().MemoryManager(); - DAddr address = - gpu_device_memory.GetAddressFromPAddr(system.DeviceMemory().GetRawPhysicalAddr(p)); + if (!gpu_device_memory) [[unlikely]] { + gpu_device_memory = &system.Host1x().MemoryManager(); + } const size_t core = system.GetCurrentHostThreadID(); auto& current_area = rasterizer_read_areas[core]; - const DAddr end_address = address + size; - if (current_area.start_address <= address && end_address <= current_area.end_address) - [[likely]] { - return; - } - current_area = system.GPU().OnCPURead(address, size); + gpu_device_memory->ApplyOpOnPointer( + p, scratch_buffers[core], [&](DAddr address) { + const DAddr end_address = address + size; + if (current_area.start_address <= address && end_address <= current_area.end_address) + [[likely]] { + return; + } + current_area = system.GPU().OnCPURead(address, size); + }); } void HandleRasterizerWrite(VAddr v_address, size_t size) { const auto* p = GetPointerImpl( v_address, []() {}, []() {}); - PAddr address = system.DeviceMemory().GetRawPhysicalAddr(p); constexpr size_t sys_core = Core::Hardware::NUM_CPU_CORES - 1; const size_t core = std::min(system.GetCurrentHostThreadID(), sys_core); // any other calls threads go to syscore. + if (!gpu_device_memory) [[unlikely]] { + gpu_device_memory = &system.Host1x().MemoryManager(); + } // Guard on sys_core; if (core == sys_core) [[unlikely]] { sys_core_guard.lock(); @@ -846,17 +852,20 @@ struct Memory::Impl { sys_core_guard.unlock(); } }); - auto& current_area = rasterizer_write_areas[core]; - PAddr subaddress = address >> YUZU_PAGEBITS; - bool do_collection = current_area.last_address == subaddress; - if (!do_collection) [[unlikely]] { - do_collection = system.GPU().OnCPUWrite(address, size); - if (!do_collection) { - return; + gpu_device_memory->ApplyOpOnPointer( + p, scratch_buffers[core], [&](DAddr address) { + auto& current_area = rasterizer_write_areas[core]; + PAddr subaddress = address >> YUZU_PAGEBITS; + bool do_collection = current_area.last_address == subaddress; + if (!do_collection) [[unlikely]] { + do_collection = system.GPU().OnCPUWrite(address, size); + if (!do_collection) { + return; + } + current_area.last_address = subaddress; } - current_area.last_address = subaddress; - } - gpu_dirty_managers[core].Collect(address, size); + gpu_dirty_managers[core].Collect(address, size); + }); } struct GPUDirtyState { @@ -872,10 +881,12 @@ struct Memory::Impl { } Core::System& system; + Tegra::MaxwellDeviceMemoryManager* gpu_device_memory{}; Common::PageTable* current_page_table = nullptr; std::array rasterizer_read_areas{}; std::array rasterizer_write_areas{}; + std::array, Core::Hardware::NUM_CPU_CORES> scratch_buffers{}; std::span gpu_dirty_managers; std::mutex sys_core_guard; diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index ca31e2fbd6..71b748c743 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -554,9 +554,8 @@ void RasterizerOpenGL::InvalidateRegion(DAddr addr, u64 size, VideoCommon::Cache } } -bool RasterizerOpenGL::OnCPUWrite(PAddr p_addr, u64 size) { +bool RasterizerOpenGL::OnCPUWrite(DAddr addr, u64 size) { MICROPROFILE_SCOPE(OpenGL_CacheManagement); - const DAddr addr = device_memory.GetAddressFromPAddr(p_addr); if (addr == 0 || size == 0) { return false; } @@ -577,9 +576,9 @@ bool RasterizerOpenGL::OnCPUWrite(PAddr p_addr, u64 size) { return false; } -void RasterizerOpenGL::OnCacheInvalidation(PAddr p_addr, u64 size) { +void RasterizerOpenGL::OnCacheInvalidation(DAddr addr, u64 size) { MICROPROFILE_SCOPE(OpenGL_CacheManagement); - const DAddr addr = device_memory.GetAddressFromPAddr(p_addr); + if (addr == 0 || size == 0) { return; } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index efcc349a0b..7db1319856 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -602,8 +602,7 @@ void RasterizerVulkan::InnerInvalidation(std::span Date: Sat, 30 Dec 2023 04:37:25 +0100 Subject: SMMU: Add Android compatibility --- src/core/device_memory_manager.inc | 7 +-- .../service/nvdrv/devices/nvhost_nvdec_common.cpp | 6 ++- src/core/memory.cpp | 62 +++++++++------------- src/core/memory.h | 4 +- src/video_core/host1x/gpu_device_memory_manager.h | 2 +- src/video_core/host1x/host1x.cpp | 2 + src/video_core/host1x/host1x.h | 1 + src/video_core/memory_manager.h | 2 +- src/video_core/query_cache/query_cache.h | 6 +-- 9 files changed, 42 insertions(+), 50 deletions(-) (limited to 'src/core/memory.cpp') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 138eb5017c..4f883cece0 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -217,9 +217,6 @@ DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memo cpu_backing_address(device_as_size >> Memory::YUZU_PAGEBITS) { impl = std::make_unique>(); cached_pages = std::make_unique(); - for (size_t i = 0; i < 1ULL << (33 - 12); i++) { - compressed_device_addr[i] = 0; - } } template @@ -517,7 +514,7 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size u64 cache_begin = 0; u64 uncache_bytes = 0; u64 cache_bytes = 0; - const auto* MarkRegionCaching = &DeviceMemoryManager::DeviceMethods::MarkRegionCaching; + const auto MarkRegionCaching = &DeviceMemoryManager::DeviceMethods::MarkRegionCaching; std::atomic_thread_fence(std::memory_order_acquire); const size_t page_end = Common::DivCeil(addr + size, Memory::YUZU_PAGESIZE); @@ -577,4 +574,4 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size } } -} // namespace Core \ No newline at end of file +} // namespace Core diff --git a/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.cpp b/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.cpp index 0b6aa99932..a50577c753 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.cpp @@ -8,6 +8,7 @@ #include "common/common_types.h" #include "common/logging/log.h" #include "core/core.h" +#include "core/hle/kernel/k_process.h" #include "core/hle/service/nvdrv/core/container.h" #include "core/hle/service/nvdrv/core/nvmap.h" #include "core/hle/service/nvdrv/core/syncpoint_manager.h" @@ -109,7 +110,7 @@ NvResult nvhost_nvdec_common::Submit(IoctlSubmit& params, std::span data, De ASSERT_OR_EXECUTE(object, return NvResult::InvalidState;); Tegra::ChCommandHeaderList cmdlist(cmd_buffer.word_count); session->process->GetMemory().ReadBlock(object->address + cmd_buffer.offset, cmdlist.data(), - cmdlist.size() * sizeof(u32)); + cmdlist.size() * sizeof(u32)); gpu.PushCommandBuffer(core.Host1xDeviceFile().fd_to_id[fd], cmdlist); } // Some games expect command_buffers to be written back @@ -135,7 +136,8 @@ NvResult nvhost_nvdec_common::GetWaitbase(IoctlGetWaitbase& params) { return NvResult::Success; } -NvResult nvhost_nvdec_common::MapBuffer(IoctlMapBuffer& params, std::span entries, DeviceFD fd) { +NvResult nvhost_nvdec_common::MapBuffer(IoctlMapBuffer& params, std::span entries, + DeviceFD fd) { const size_t num_entries = std::min(params.num_entries, static_cast(entries.size())); for (size_t i = 0; i < num_entries; i++) { DAddr pin_address = nvmap.PinHandle(entries[i].map_handle, sessions[fd], true); diff --git a/src/core/memory.cpp b/src/core/memory.cpp index f126840cbf..1c218566fd 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -44,8 +44,7 @@ bool AddressSpaceContains(const Common::PageTable& table, const Common::ProcessA // from outside classes. This also allows modification to the internals of the memory // subsystem without needing to rebuild all files that make use of the memory interface. struct Memory::Impl { - explicit Impl(Core::System& system_) - : system{system_} {} + explicit Impl(Core::System& system_) : system{system_} {} void SetCurrentPageTable(Kernel::KProcess& process) { current_page_table = &process.GetPageTable().GetImpl(); @@ -640,18 +639,6 @@ struct Memory::Impl { LOG_DEBUG(HW_Memory, "Mapping {:016X} onto {:016X}-{:016X}", GetInteger(target), base * YUZU_PAGESIZE, (base + size) * YUZU_PAGESIZE); - // During boot, current_page_table might not be set yet, in which case we need not flush - /*if (system.IsPoweredOn()) { - auto& gpu = system.GPU(); - for (u64 i = 0; i < size; i++) { - const auto page = base + i; - if (page_table.pointers[page].Type() == Common::PageType::RasterizerCachedMemory) { - - gpu.FlushAndInvalidateRegion(page << YUZU_PAGEBITS, YUZU_PAGESIZE); - } - } - }*/ - const auto end = base + size; ASSERT_MSG(end <= page_table.pointers.size(), "out of range mapping at {:016X}", base + page_table.pointers.size()); @@ -823,8 +810,7 @@ struct Memory::Impl { } const size_t core = system.GetCurrentHostThreadID(); auto& current_area = rasterizer_read_areas[core]; - gpu_device_memory->ApplyOpOnPointer( - p, scratch_buffers[core], [&](DAddr address) { + gpu_device_memory->ApplyOpOnPointer(p, scratch_buffers[core], [&](DAddr address) { const DAddr end_address = address + size; if (current_area.start_address <= address && end_address <= current_area.end_address) [[likely]] { @@ -852,8 +838,7 @@ struct Memory::Impl { sys_core_guard.unlock(); } }); - gpu_device_memory->ApplyOpOnPointer( - p, scratch_buffers[core], [&](DAddr address) { + gpu_device_memory->ApplyOpOnPointer(p, scratch_buffers[core], [&](DAddr address) { auto& current_area = rasterizer_write_areas[core]; PAddr subaddress = address >> YUZU_PAGEBITS; bool do_collection = current_area.last_address == subaddress; @@ -872,12 +857,25 @@ struct Memory::Impl { PAddr last_address; }; - void InvalidateRegion(Common::ProcessAddress dest_addr, size_t size) { - system.GPU().InvalidateRegion(GetInteger(dest_addr), size); - } - - void FlushRegion(Common::ProcessAddress dest_addr, size_t size) { - system.GPU().FlushRegion(GetInteger(dest_addr), size); + void InvalidateGPUMemory(u8* p, size_t size) { + constexpr size_t sys_core = Core::Hardware::NUM_CPU_CORES - 1; + const size_t core = std::min(system.GetCurrentHostThreadID(), + sys_core); // any other calls threads go to syscore. + if (!gpu_device_memory) [[unlikely]] { + gpu_device_memory = &system.Host1x().MemoryManager(); + } + // Guard on sys_core; + if (core == sys_core) [[unlikely]] { + sys_core_guard.lock(); + } + SCOPE_EXIT({ + if (core == sys_core) [[unlikely]] { + sys_core_guard.unlock(); + } + }); + auto& gpu = system.GPU(); + gpu_device_memory->ApplyOpOnPointer( + p, scratch_buffers[core], [&](DAddr address) { gpu.InvalidateRegion(address, size); }); } Core::System& system; @@ -1081,14 +1079,6 @@ void Memory::MarkRegionDebug(Common::ProcessAddress vaddr, u64 size, bool debug) impl->MarkRegionDebug(GetInteger(vaddr), size, debug); } -void Memory::InvalidateRegion(Common::ProcessAddress dest_addr, size_t size) { - impl->InvalidateRegion(dest_addr, size); -} - -void Memory::FlushRegion(Common::ProcessAddress dest_addr, size_t size) { - impl->FlushRegion(dest_addr, size); -} - bool Memory::InvalidateNCE(Common::ProcessAddress vaddr, size_t size) { [[maybe_unused]] bool mapped = true; [[maybe_unused]] bool rasterizer = false; @@ -1100,10 +1090,10 @@ bool Memory::InvalidateNCE(Common::ProcessAddress vaddr, size_t size) { GetInteger(vaddr)); mapped = false; }, - [&] { - impl->system.GPU().InvalidateRegion(GetInteger(vaddr), size); - rasterizer = true; - }); + [&] { rasterizer = true; }); + if (rasterizer) { + impl->InvalidateGPUMemory(ptr, size); + } #ifdef __linux__ if (!rasterizer && mapped) { diff --git a/src/core/memory.h b/src/core/memory.h index 47ca6a35a8..9d29cfd3f1 100644 --- a/src/core/memory.h +++ b/src/core/memory.h @@ -486,10 +486,10 @@ public: void MarkRegionDebug(Common::ProcessAddress vaddr, u64 size, bool debug); void SetGPUDirtyManagers(std::span managers); - void InvalidateRegion(Common::ProcessAddress dest_addr, size_t size); + bool InvalidateNCE(Common::ProcessAddress vaddr, size_t size); + bool InvalidateSeparateHeap(void* fault_address); - void FlushRegion(Common::ProcessAddress dest_addr, size_t size); private: Core::System& system; diff --git a/src/video_core/host1x/gpu_device_memory_manager.h b/src/video_core/host1x/gpu_device_memory_manager.h index 6c78588483..9ccd84b9a2 100644 --- a/src/video_core/host1x/gpu_device_memory_manager.h +++ b/src/video_core/host1x/gpu_device_memory_manager.h @@ -17,7 +17,7 @@ struct MaxwellDeviceTraits { static constexpr bool supports_pinning = false; static constexpr size_t device_virtual_bits = 34; using DeviceInterface = typename VideoCore::RasterizerInterface; - using DeviceMethods = typename MaxwellDeviceMethods; + using DeviceMethods = MaxwellDeviceMethods; }; using MaxwellDeviceMemoryManager = Core::DeviceMemoryManager; diff --git a/src/video_core/host1x/host1x.cpp b/src/video_core/host1x/host1x.cpp index b7f9a08cf1..c4c7a5883d 100644 --- a/src/video_core/host1x/host1x.cpp +++ b/src/video_core/host1x/host1x.cpp @@ -13,6 +13,8 @@ Host1x::Host1x(Core::System& system_) memory_manager(system.DeviceMemory()), gmmu_manager{system, memory_manager, 32, 12}, allocator{std::make_unique>(1 << 12)} {} +Host1x::~Host1x() = default; + } // namespace Host1x } // namespace Tegra diff --git a/src/video_core/host1x/host1x.h b/src/video_core/host1x/host1x.h index 13c37e6b4b..d72d97b7b3 100644 --- a/src/video_core/host1x/host1x.h +++ b/src/video_core/host1x/host1x.h @@ -21,6 +21,7 @@ namespace Host1x { class Host1x { public: explicit Host1x(Core::System& system); + ~Host1x(); SyncpointManager& GetSyncpointManager() { return syncpoint_manager; diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index 6b2cd7efb9..00d64dcce0 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -68,7 +68,7 @@ public: if (!address) { return {}; } - return memory.GetPointer(*address); + return memory.GetPointer(*address); } template diff --git a/src/video_core/query_cache/query_cache.h b/src/video_core/query_cache/query_cache.h index 508afb10a1..b5e90cf8c2 100644 --- a/src/video_core/query_cache/query_cache.h +++ b/src/video_core/query_cache/query_cache.h @@ -256,8 +256,8 @@ void QueryCacheBase::CounterReport(GPUVAddr addr, QueryType counter_type return std::make_pair(cur_addr >> Core::Memory::YUZU_PAGEBITS, static_cast(cur_addr & Core::Memory::YUZU_PAGEMASK)); }; - u8* pointer = impl->device_memory.GetPointer(cpu_addr); - u8* pointer_timestamp = impl->device_memory.GetPointer(cpu_addr + 8); + u8* pointer = impl->device_memory.template GetPointer(cpu_addr); + u8* pointer_timestamp = impl->device_memory.template GetPointer(cpu_addr + 8); bool is_synced = !Settings::IsGPULevelHigh() && is_fence; std::function operation([this, is_synced, streamer, query_base = query, query_location, pointer, pointer_timestamp] { @@ -561,7 +561,7 @@ bool QueryCacheBase::SemiFlushQueryDirty(QueryCacheBase::QueryLo } if (True(query_base->flags & QueryFlagBits::IsFinalValueSynced) && False(query_base->flags & QueryFlagBits::IsGuestSynced)) { - auto* ptr = impl->device_memory.GetPointer(query_base->guest_address); + auto* ptr = impl->device_memory.template GetPointer(query_base->guest_address); if (True(query_base->flags & QueryFlagBits::HasTimestamp)) { std::memcpy(ptr, &query_base->value, sizeof(query_base->value)); return false; -- cgit v1.2.3