aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/video_core/buffer_cache/buffer_cache.h191
-rw-r--r--src/video_core/buffer_cache/buffer_cache_base.h51
-rw-r--r--src/video_core/dma_pusher.cpp98
-rw-r--r--src/video_core/dma_pusher.h5
-rw-r--r--src/video_core/gpu_split.cpp30
-rw-r--r--src/video_core/gpu_split.h10
-rw-r--r--src/video_core/nx2.h32
-rw-r--r--src/video_core/rasterizer_interface.h14
-rw-r--r--src/video_core/renderer_vulkan/vk_buffer_cache.cpp4
-rw-r--r--src/video_core/renderer_vulkan/vk_buffer_cache.h3
-rw-r--r--src/video_core/renderer_vulkan/vk_rasterizer.cpp40
-rw-r--r--src/video_core/renderer_vulkan/vk_rasterizer.h6
12 files changed, 478 insertions, 6 deletions
diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h
index 53bbae7610..cd176dec27 100644
--- a/src/video_core/buffer_cache/buffer_cache.h
+++ b/src/video_core/buffer_cache/buffer_cache.h
@@ -174,6 +174,7 @@ void BufferCache<P>::UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t size
template <class P>
void BufferCache<P>::WriteMemory(DAddr device_addr, u64 size) {
+ Nx2NoteListHostWrite(device_addr, size);
if (IsRegionGpuModified(device_addr, size)) {
ClearDownload(device_addr, size);
gpu_modified_ranges.Subtract(device_addr, size);
@@ -194,6 +195,7 @@ void BufferCache<P>::CachedWriteMemory(DAddr device_addr, u64 size) {
return;
}
+ Nx2NoteListHostWrite(device_addr, size);
tmp_buffer.resize_destructive(size);
device_memory.ReadBlockUnsafe(device_addr, tmp_buffer.data(), size);
@@ -301,6 +303,7 @@ bool BufferCache<P>::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am
gpu_modified_ranges.Add(pair.first, pair.second);
if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) {
Tegra::Nx2MarkGpuWritten(pair.first, pair.second);
+ Nx2NoteListWrite(buffer_b, pair.first, pair.second);
}
}
const auto& copy = copies[0];
@@ -1660,6 +1663,194 @@ void BufferCache<P>::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3
}
if (Tegra::g_nx2_ampere.load(std::memory_order_relaxed)) {
Tegra::Nx2MarkGpuWritten(device_addr, size);
+ Nx2NoteListWrite(buffer_id, device_addr, size);
+ }
+}
+
+template <class P>
+void BufferCache<P>::Nx2NoteListWrite(BufferId buffer_id, DAddr device_addr, u64 size) {
+ if constexpr (!IS_OPENGL) {
+ if (size == 0 || !Tegra::g_nx2_has_command_list_pages.load(std::memory_order_relaxed)) {
+ return;
+ }
+ constexpr u64 page_bits = Tegra::Nx2WrittenPageBits;
+ const u64 last = (std::min<u64>)((device_addr + size - 1) >> page_bits,
+ Tegra::g_nx2_command_list_pages.size() * 64 - 1);
+ for (u64 page = device_addr >> page_bits; page <= last;) {
+ // Storage buffers can be large: skip 64 pages at a time where none is a list page.
+ const u64 word =
+ Tegra::g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >>
+ (page % 64);
+ if (word == 0) {
+ page = (page / 64 + 1) * 64;
+ continue;
+ }
+ if (word & 1) {
+ const DAddr page_addr = page << page_bits;
+ {
+ std::scoped_lock lock{nx2_list_mutex};
+ Nx2ListPage& entry = nx2_list_pages[page_addr];
+ ++entry.writes;
+ if (!nx2_collect_list_copies && !entry.draw_written) {
+ entry.draw_written = true;
+ LOG_INFO(HW_GPU, "Command list page {:#x} is written by draws too: its "
+ "lists drain the split",
+ page_addr);
+ }
+ }
+ if (nx2_collect_list_copies) {
+ nx2_list_copy_requests.emplace_back(buffer_id, page_addr);
+ }
+ }
+ ++page;
+ }
+ }
+}
+
+template <class P>
+bool BufferCache<P>::Nx2CopyListPages(u64 tick) {
+ if constexpr (IS_OPENGL) {
+ return false;
+ } else {
+ if (nx2_list_copy_requests.empty()) {
+ return false;
+ }
+ std::ranges::sort(nx2_list_copy_requests, {}, &std::pair<BufferId, DAddr>::second);
+ constexpr DAddr page_size = DAddr{1} << Tegra::Nx2WrittenPageBits;
+ bool recorded = false;
+ for (size_t i = 0; i < nx2_list_copy_requests.size(); ++i) {
+ const auto [buffer_id, page] = nx2_list_copy_requests[i];
+ if (i + 1 < nx2_list_copy_requests.size() &&
+ nx2_list_copy_requests[i + 1].second == page) {
+ continue;
+ }
+ Buffer& buffer = slot_buffers[buffer_id];
+ const DAddr begin = (std::max<DAddr>)(page, buffer.CpuAddr());
+ const DAddr end =
+ (std::min<DAddr>)(page + page_size, buffer.CpuAddr() + buffer.SizeBytes());
+ if (begin >= end) {
+ continue;
+ }
+ Nx2Staging staging = runtime.DownloadStagingBuffer(page_size, true);
+ const BufferCopy copy{
+ .src_offset = buffer.Offset(begin),
+ .dst_offset = staging.offset + (begin - page),
+ .size = end - begin,
+ };
+ buffer.MarkUsage(copy.src_offset, copy.size);
+ runtime.CopyBuffer(staging.buffer, buffer, std::span{&copy, 1}, true);
+ recorded = true;
+ std::scoped_lock lock{nx2_list_mutex};
+ Nx2ListPage& entry = nx2_list_pages[page];
+ if (entry.copy) {
+ nx2_retired_list_copies.emplace_back(entry.copy_tick, std::move(*entry.copy));
+ }
+ entry.copy = staging;
+ entry.copy_writes = entry.writes;
+ entry.copy_tick = tick;
+ entry.copy_begin = static_cast<u32>(begin - page);
+ entry.copy_end = static_cast<u32>(end - page);
+ }
+ nx2_list_copy_requests.clear();
+ std::erase_if(nx2_retired_list_copies, [this](auto& retired) {
+ if (!runtime.IsFree(retired.first)) {
+ return false;
+ }
+ runtime.FreeDeferredStagingBuffer(retired.second);
+ return true;
+ });
+ return recorded;
+ }
+}
+
+template <class P>
+auto BufferCache<P>::Nx2ReadListCopy(DAddr device_addr, std::span<u8> out) -> Nx2ListRead {
+ if constexpr (IS_OPENGL) {
+ return Nx2ListRead::Unknown;
+ } else {
+ constexpr DAddr page_size = DAddr{1} << Tegra::Nx2WrittenPageBits;
+ const DAddr page = device_addr & ~(page_size - 1);
+ const u64 offset = device_addr - page;
+ if (offset + out.size() > page_size) {
+ return Nx2ListRead::Unknown;
+ }
+ const auto current = [&](const Nx2ListPage& entry) {
+ return entry.copy && !entry.draw_written && entry.copy_writes == entry.writes &&
+ offset >= entry.copy_begin && offset + out.size() <= entry.copy_end;
+ };
+ u64 tick{};
+ u64 writes{};
+ {
+ std::scoped_lock lock{nx2_list_mutex};
+ const auto it = nx2_list_pages.find(page);
+ if (it == nx2_list_pages.end()) {
+ return Nx2ListRead::Unknown;
+ }
+ const Nx2ListPage& entry = it->second;
+ // Pages the CPU writes the lists of (GPU-written once, long before) stay synced.
+ if (!entry.draw_written && entry.synced_writes == entry.writes) {
+ return Nx2ListRead::Synced;
+ }
+ if (!current(entry)) {
+ return Nx2ListRead::Unknown;
+ }
+ // The lists after the first of a batch find the copy done: read it at once.
+ if (runtime.IsFree(entry.copy_tick)) {
+ std::memcpy(out.data(), entry.copy->mapped_span.data() + offset, out.size());
+ return Nx2ListRead::Copied;
+ }
+ tick = entry.copy_tick;
+ writes = entry.copy_writes;
+ }
+ // Submitted with the work that wrote the page (Nx2CopyListPages's caller flushes).
+ runtime.WaitSubmittedTick(tick);
+ std::scoped_lock lock{nx2_list_mutex};
+ const auto it = nx2_list_pages.find(page);
+ if (it == nx2_list_pages.end() || !current(it->second) || it->second.copy_tick != tick ||
+ it->second.copy_writes != writes) {
+ return Nx2ListRead::Unknown;
+ }
+ std::memcpy(out.data(), it->second.copy->mapped_span.data() + offset, out.size());
+ return Nx2ListRead::Copied;
+ }
+}
+
+template <class P>
+void BufferCache<P>::Nx2NoteListSynced(DAddr page) {
+ if constexpr (!IS_OPENGL) {
+ std::scoped_lock lock{nx2_list_mutex};
+ Nx2ListPage& entry = nx2_list_pages[page];
+ entry.synced_writes = entry.writes;
+ }
+}
+
+template <class P>
+void BufferCache<P>::Nx2NoteListHostWrite(DAddr device_addr, u64 size) {
+ if constexpr (!IS_OPENGL) {
+ if (size == 0 || !Tegra::g_nx2_has_command_list_pages.load(std::memory_order_relaxed)) {
+ return;
+ }
+ constexpr u64 page_bits = Tegra::Nx2WrittenPageBits;
+ const u64 last = (std::min<u64>)((device_addr + size - 1) >> page_bits,
+ Tegra::g_nx2_command_list_pages.size() * 64 - 1);
+ for (u64 page = device_addr >> page_bits; page <= last; ++page) {
+ if (!((Tegra::g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >>
+ (page % 64)) &
+ 1)) {
+ continue;
+ }
+ std::scoped_lock lock{nx2_list_mutex};
+ const auto it = nx2_list_pages.find(page << page_bits);
+ if (it == nx2_list_pages.end()) {
+ continue;
+ }
+ Nx2ListPage& entry = it->second;
+ const bool synced = entry.synced_writes == entry.writes;
+ ++entry.writes;
+ if (synced) {
+ entry.synced_writes = entry.writes;
+ }
+ }
}
}
diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h
index 47f342932d..1f0c6cf10f 100644
--- a/src/video_core/buffer_cache/buffer_cache_base.h
+++ b/src/video_core/buffer_cache/buffer_cache_base.h
@@ -14,7 +14,9 @@
#include <memory>
#include <mutex>
#include <numeric>
+#include <optional>
#include <span>
+#include <unordered_map>
#include <vector>
#include "common/container/unordered_map.h"
@@ -321,6 +323,28 @@ public:
bool DMAClear(GPUVAddr src_address, u64 amount, u32 value);
+ /// Switch 2 GPU-written command lists (nx2.h, g_nx2_command_list_pages). While collecting (a
+ /// compute dispatch or a DMA copy being recorded), writes to command list pages are noted;
+ /// Nx2CopyListPages then records copies of those pages into host memory after the work.
+ void Nx2CollectListCopies(bool collect) {
+ nx2_collect_list_copies = collect;
+ if (!collect) {
+ nx2_list_copy_requests.clear();
+ }
+ }
+ /// True when it recorded copies, which the GPU completes at `tick` (the caller submits).
+ bool Nx2CopyListPages(u64 tick);
+ enum class Nx2ListRead { Unknown, Copied, Synced };
+ /// Front thread: Synced when guest memory holds the page `device_addr` lies in (flushed and
+ /// written by no GPU work since), else Copied with `out` filled from the copy of its last
+ /// write once the GPU is done with it. Unknown otherwise: the caller drains and flushes.
+ Nx2ListRead Nx2ReadListCopy(DAddr device_addr, std::span<u8> out);
+ /// Front thread, the split drained: the command list page was flushed to guest memory.
+ void Nx2NoteListSynced(DAddr page);
+ /// CPU and inline writes: they reach guest memory, so a synced page stays synced, but its copy
+ /// no longer holds the page.
+ void Nx2NoteListHostWrite(DAddr device_addr, u64 size);
+
/// Return true when a CPU region is modified from the GPU
[[nodiscard]] bool IsRegionGpuModified(DAddr addr, size_t size);
@@ -440,6 +464,10 @@ private:
void MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size, bool needs_sync = false);
+ /// Counts a GPU write to the command list pages in the range (invalidating their copies) and,
+ /// while collecting, asks for new copies.
+ void Nx2NoteListWrite(BufferId buffer_id, DAddr device_addr, u64 size);
+
[[nodiscard]] BufferId FindBuffer(DAddr device_addr, u32 size, bool sparse_compatible);
void SynchronizeBufferWrites(Buffer& buffer);
@@ -604,6 +632,29 @@ private:
// Covers the device address space (MaxwellDeviceTraits::device_virtual_bits).
std::array<BufferId, ((1ULL << 36) >> CACHING_PAGEBITS)> page_table;
Common::ScratchBuffer<u8> tmp_buffer;
+
+ /// A command list page (Nx2CopyListPages): the GPU writes noted to it and the last copy.
+ using Nx2Staging = decltype(std::declval<Runtime&>().DownloadStagingBuffer(size_t{}, true));
+ struct Nx2ListPage {
+ u64 writes{}; ///< GPU writes noted to the page
+ u64 copy_writes{}; ///< `writes` when the copy was made: the copy is current while equal
+ u64 copy_tick{}; ///< GPU tick that completes the copy
+ /// `writes` when the front last flushed the page: guest memory holds it while equal
+ u64 synced_writes{~u64{0}};
+ u32 copy_begin{}; ///< Bytes of the page the copy holds (a buffer can end inside it)
+ u32 copy_end{};
+ /// Written outside a compute dispatch or a DMA copy (a draw's storage writes) at least
+ /// once: the front can't tell from what it sent whether such a write is still queued, so
+ /// the page's lists always drain the split instead of reading a copy.
+ bool draw_written{};
+ std::optional<Nx2Staging> copy;
+ };
+ std::mutex nx2_list_mutex; ///< nx2_list_pages: the split's front reads the copies
+ std::unordered_map<DAddr, Nx2ListPage> nx2_list_pages;
+ /// Copies replaced while the GPU may still write them: freed once it is past their tick.
+ std::vector<std::pair<u64, Nx2Staging>> nx2_retired_list_copies;
+ boost::container::small_vector<std::pair<BufferId, DAddr>, 8> nx2_list_copy_requests;
+ bool nx2_collect_list_copies{};
};
} // namespace VideoCommon
diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp
index 1144488c91..6b93bb7f60 100644
--- a/src/video_core/dma_pusher.cpp
+++ b/src/video_core/dma_pusher.cpp
@@ -9,6 +9,7 @@
#include <cstdlib>
#include <cstdio>
+#include "common/scope_exit.h"
#include "common/settings.h"
#include "core/core.h"
#include "core/memory.h"
@@ -178,6 +179,11 @@ bool DmaPusher::Step() {
RecordParsedEntry(header, commands);
command_resume_offset +=
ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset);
+ } else if (ReadGpuWrittenCommands(dma_state.dma_get, static_cast<u32>(header.size))) {
+ const std::span<const CommandHeader> commands{command_headers.data(), header.size};
+ RecordParsedEntry(header, commands);
+ command_resume_offset +=
+ ProcessCommands(commands.subspan(command_resume_offset), command_resume_offset);
} else {
FlushGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32));
Tegra::Memory::GpuGuestMemory<Tegra::CommandHeader, Tegra::Memory::GuestMemoryFlags::UnsafeRead>headers(memory_manager, dma_state.dma_get, header.size, &command_headers);
@@ -382,6 +388,15 @@ void DmaPusher::CallMethod(u32 argument) {
ReportUnboundSubchannel();
return;
}
+ // Compute dispatches and DMA copies write the command lists ReadGpuWrittenCommands reads.
+ const auto engine_type = subchannel_type[dma_state.subchannel];
+ const bool writer = engine_type == Engines::EngineTypes::KeplerCompute ||
+ engine_type == Engines::EngineTypes::MaxwellDMA;
+ SCOPE_EXIT {
+ if (writer) {
+ split.NoteWriter();
+ }
+ };
if (split.Active()) {
const GPUVAddr segment = dma_state.dma_get + dma_state.dma_word_offset;
if (split.FrontMacros()) {
@@ -432,6 +447,15 @@ void DmaPusher::CallMultiMethod(const u32* base_start, u32 num_methods) {
ReportUnboundSubchannel();
return;
}
+ // Compute dispatches and DMA copies write the command lists ReadGpuWrittenCommands reads.
+ const auto engine_type = subchannel_type[dma_state.subchannel];
+ const bool writer = engine_type == Engines::EngineTypes::KeplerCompute ||
+ engine_type == Engines::EngineTypes::MaxwellDMA;
+ SCOPE_EXIT {
+ if (writer) {
+ split.NoteWriter();
+ }
+ };
if (split.Active()) {
const GPUVAddr segment = dma_state.dma_get + dma_state.dma_word_offset;
if (split.FrontMacros()) {
@@ -476,18 +500,86 @@ bool DmaPusher::IsGpuWrittenCommands(GPUVAddr address, size_t size) {
return false;
}
+bool DmaPusher::ReadGpuWrittenCommands(GPUVAddr address, u32 count) {
+ // The pages are marked as command list pages: from the next write on, the buffer cache copies
+ // them to host memory right after the dispatch or copy that writes them and submits. The
+ // front then waits only for the back to run that work and for the GPU to finish the copy, not
+ // for everything queued and a cache flush (FlushGpuWrittenCommands: ~400 GPU round trips a
+ // second in a Mario Kart World race, during which the back had nothing to do). Pages
+ // FlushGpuWrittenCommands flushed whole and no GPU work wrote since are read from guest
+ // memory.
+ if (!Tegra::g_nx2_ampere.load(std::memory_order_relaxed) || count == 0 ||
+ rasterizer == nullptr) {
+ return false;
+ }
+ const GPUVAddr end = address + u64{count} * sizeof(u32);
+ bool written = false;
+ for (GPUVAddr page = address & ~GPUVAddr{0xfff}; page < end; page += 0x1000) {
+ if (const auto device = memory_manager.GpuToCpuAddress(page);
+ device && Tegra::Nx2IsGpuWritten(*device)) {
+ Tegra::Nx2MarkCommandListPage(*device);
+ written = true;
+ }
+ }
+ if (!written) {
+ return false;
+ }
+ if (auto& split = VideoCommon::GpuSplit::Get(); split.Active()) {
+ split.WaitForWriters();
+ }
+ using GpuWrittenList = VideoCore::RasterizerInterface::GpuWrittenList;
+ command_headers.resize_destructive(count);
+ u8* const out = reinterpret_cast<u8*>(command_headers.data());
+ for (GPUVAddr chunk = address; chunk < end;) {
+ const GPUVAddr chunk_end = (std::min)(end, (chunk & ~GPUVAddr{0xfff}) + 0x1000);
+ const std::span<u8> part{out + (chunk - address), static_cast<size_t>(chunk_end - chunk)};
+ const auto device = memory_manager.GpuToCpuAddress(chunk);
+ if (!device) {
+ return false;
+ }
+ GpuWrittenList where = GpuWrittenList::Synced;
+ if (Tegra::Nx2IsGpuWritten(*device)) {
+ where = rasterizer->ReadGpuWrittenCommandList(*device, part);
+ }
+ if (where == GpuWrittenList::Unknown) {
+ return false;
+ }
+ if (where == GpuWrittenList::Synced) {
+ memory_manager.ReadBlockUnsafe(chunk, part.data(), part.size());
+ }
+ chunk = chunk_end;
+ }
+ return true;
+}
+
void DmaPusher::FlushGpuWrittenCommands(GPUVAddr address, size_t size) {
// Title 1 runs small command lists its culling dispatches fill in each frame. Accurate GPU
// mode reads every command list through the caches (safe reads); here only the ones on pages
// the GPU has written are, or their draws run with an older frame's words (trees vanish).
+ // These are the lists ReadGpuWrittenCommands can't take from a copy or guest memory (a page's
+ // first list, or pages draws write too).
if (!IsGpuWrittenCommands(address, size)) {
return;
}
// The caches belong to the split's back: it first runs what the front queued (the dispatch
- // that writes these words among it).
+ // that writes these words among it). The whole pages are flushed: their next lists read
+ // guest memory until GPU work writes them again.
VideoCommon::GpuSplit::Get().Drain();
- if (memory_manager.IsMemoryDirty(address, size, VideoCommon::CacheType::BufferCache)) {
- memory_manager.FlushRegion(address, size, VideoCommon::CacheType::BufferCache);
+ const GPUVAddr first_page = address & ~GPUVAddr{0xfff};
+ const GPUVAddr end_page = (address + size + 0xfff) & ~GPUVAddr{0xfff};
+ if (memory_manager.IsMemoryDirty(first_page, end_page - first_page,
+ VideoCommon::CacheType::BufferCache)) {
+ memory_manager.FlushRegion(first_page, end_page - first_page,
+ VideoCommon::CacheType::BufferCache);
+ }
+ if (rasterizer == nullptr) {
+ return;
+ }
+ for (GPUVAddr page = first_page; page < end_page; page += 0x1000) {
+ if (const auto device = memory_manager.GpuToCpuAddress(page);
+ device && Tegra::Nx2IsCommandListPage(*device)) {
+ rasterizer->NoteGpuWrittenCommandListSynced(*device);
+ }
}
}
diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h
index e9dd5e2b6b..289786bb9e 100644
--- a/src/video_core/dma_pusher.h
+++ b/src/video_core/dma_pusher.h
@@ -188,6 +188,11 @@ public:
void RecordParsedEntry(const CommandListHeader& header, std::span<const CommandHeader> commands);
/// Whether the entry lies on device pages the GPU has written (NX2).
bool IsGpuWrittenCommands(GPUVAddr address, size_t size);
+ /// Reads a command list on pages the GPU wrote into command_headers, from the copies the
+ /// buffer cache made after the work that wrote them or from guest memory for pages flushed
+ /// since. False when not all are current (or the list isn't GPU-written): the caller drains,
+ /// flushes and reads guest memory.
+ bool ReadGpuWrittenCommands(GPUVAddr address, u32 count);
/// Brings command list words the GPU wrote (still only in host buffers) to guest memory.
void FlushGpuWrittenCommands(GPUVAddr address, size_t size);
diff --git a/src/video_core/gpu_split.cpp b/src/video_core/gpu_split.cpp
index ac0d3f25f1..ef0cdae123 100644
--- a/src/video_core/gpu_split.cpp
+++ b/src/video_core/gpu_split.cpp
@@ -138,6 +138,7 @@ void GpuSplit::Stop() {
// never ran, it waited on its syncpoint forever), and the engines it held were freed.
ring.reset();
write_position = 0;
+ last_writer_position = 0;
published_local = 0;
consumed_cache = 0;
publish_every = PublishEvery;
@@ -216,6 +217,28 @@ void GpuSplit::Drain() {
}
}
+void GpuSplit::WaitForWriters() {
+ const u64 target = std::min(last_writer_position, write_position);
+ consumed_cache = consumed.load(std::memory_order_acquire);
+ if (consumed_cache >= target) {
+ return;
+ }
+ wait_target.store(target, std::memory_order_seq_cst);
+ Publish();
+ for (u32 spins = 0;; ++spins) {
+ consumed_cache = consumed.load(std::memory_order_acquire);
+ if (consumed_cache >= target) {
+ wait_target.store(~u64{0}, std::memory_order_relaxed);
+ return;
+ }
+ if (spins < 16384) {
+ Pause();
+ } else {
+ std::this_thread::yield();
+ }
+ }
+}
+
void GpuSplit::SwitchEngine(Tegra::Engines::EngineInterface* engine) {
front_engine = engine;
u64* const entry = Reserve(2);
@@ -346,8 +369,11 @@ void GpuSplit::BackLoop(std::stop_token stop_token) {
const size_t index = read_position & RingMask;
const size_t size = Execute(&ring[index]);
read_position += size != 0 ? size : RingWords - index;
- // Hand ring space back now and then while working through a long batch.
- if (read_position - last_store >= 4096 && read_position != available) {
+ // Hand ring space back now and then while working through a long batch, and at once
+ // when the front waits for this position (WaitForWriters).
+ if ((read_position - last_store >= 4096 ||
+ read_position >= wait_target.load(std::memory_order_relaxed)) &&
+ read_position != available) {
consumed.store(read_position, std::memory_order_release);
last_store = read_position;
}
diff --git a/src/video_core/gpu_split.h b/src/video_core/gpu_split.h
index d6f91e4141..c7c7118481 100644
--- a/src/video_core/gpu_split.h
+++ b/src/video_core/gpu_split.h
@@ -100,6 +100,13 @@ public:
/// Waits until the back executed everything sent so far.
void Drain();
+ /// Compute or copy work was just sent: what can write GPU-written command lists.
+ void NoteWriter() {
+ last_writer_position = write_position;
+ }
+ /// Waits until the back executed the compute and copy work sent so far (not what came after).
+ void WaitForWriters();
+
private:
enum class Kind : u64 {
Wrap, ///< The rest of the ring is unused: continue at its start
@@ -156,6 +163,7 @@ private:
// Front side.
u64 write_position{};
+ u64 last_writer_position{};
u64 published_local{};
u64 consumed_cache{};
/// Words between publishes: small right after a drain, while the back has nothing to do.
@@ -169,6 +177,8 @@ private:
alignas(64) std::atomic<u64> published{};
alignas(64) std::atomic<u64> consumed{};
+ /// A position the front waits for (WaitForWriters): the back stores `consumed` once past it.
+ alignas(64) std::atomic<u64> wait_target{~u64{0}};
alignas(64) std::atomic<bool> back_waiting{};
std::atomic<u32> wake{};
std::jthread back_thread;
diff --git a/src/video_core/nx2.h b/src/video_core/nx2.h
index db8c92adfd..92714d52d2 100644
--- a/src/video_core/nx2.h
+++ b/src/video_core/nx2.h
@@ -43,11 +43,43 @@ inline void Nx2MarkGpuWritten(u64 device_address, u64 size) {
}
}
+// GPU-written device pages the front has read command lists from (same layout as above). The
+// buffer cache copies these pages into host memory right after the compute or copy work that
+// writes them and submits, so the front reads the copy once the GPU is done instead of draining
+// the split and flushing the caches (BufferCache::Nx2ReadListCopy).
+inline std::array<std::atomic<u64>, (u64{1} << (36 - Nx2WrittenPageBits)) / 64> g_nx2_command_list_pages{};
+inline std::atomic<bool> g_nx2_has_command_list_pages{false};
+
+inline void Nx2MarkCommandListPage(u64 device_address) {
+ const u64 page = device_address >> Nx2WrittenPageBits;
+ if (page >= g_nx2_command_list_pages.size() * 64) {
+ return;
+ }
+ auto& word = g_nx2_command_list_pages[page / 64];
+ const u64 bit = u64{1} << (page % 64);
+ if ((word.load(std::memory_order_relaxed) & bit) == 0) {
+ word.fetch_or(bit, std::memory_order_relaxed);
+ g_nx2_has_command_list_pages.store(true, std::memory_order_relaxed);
+ }
+}
+
+[[nodiscard]] inline bool Nx2IsCommandListPage(u64 device_address) {
+ const u64 page = device_address >> Nx2WrittenPageBits;
+ if (page >= g_nx2_command_list_pages.size() * 64) {
+ return false;
+ }
+ return (g_nx2_command_list_pages[page / 64].load(std::memory_order_relaxed) >> (page % 64)) & 1;
+}
+
/// A new emulation session starts: no page has been written by its GPU yet.
inline void Nx2ClearGpuWrittenPages() {
for (auto& word : g_nx2_gpu_written_pages) {
word.store(0, std::memory_order_relaxed);
}
+ for (auto& word : g_nx2_command_list_pages) {
+ word.store(0, std::memory_order_relaxed);
+ }
+ g_nx2_has_command_list_pages.store(false, std::memory_order_relaxed);
}
[[nodiscard]] inline bool Nx2IsGpuWritten(u64 device_address) {
diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h
index 8486d44ac9..e9eaa2b3c2 100644
--- a/src/video_core/rasterizer_interface.h
+++ b/src/video_core/rasterizer_interface.h
@@ -94,6 +94,20 @@ public:
virtual void FlushRegion(DAddr addr, u64 size,
VideoCommon::CacheType which = VideoCommon::CacheType::All) = 0;
+ /// Switch 2, from the GPU thread's front: where a GPU-written command list's words are.
+ enum class GpuWrittenList {
+ Unknown, ///< Neither: the caller drains, flushes the page and notes it synced
+ Copied, ///< `out` (within one 4 KiB page) is filled from the copy made after its writer
+ Synced, ///< Guest memory holds the page: no GPU work wrote it since it was flushed
+ };
+ virtual GpuWrittenList ReadGpuWrittenCommandList([[maybe_unused]] DAddr addr,
+ [[maybe_unused]] std::span<u8> out) {
+ return GpuWrittenList::Unknown;
+ }
+
+ /// Switch 2: the front flushed this command list page with the split drained.
+ virtual void NoteGpuWrittenCommandListSynced([[maybe_unused]] DAddr page) {}
+
/// Check if the the specified memory area requires flushing to CPU Memory.
virtual bool MustFlushRegion(DAddr addr, u64 size,
VideoCommon::CacheType which = VideoCommon::CacheType::All) = 0;
diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp
index fe0ab3a28e..2449265509 100644
--- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp
@@ -453,6 +453,10 @@ void BufferCacheRuntime::Wait(u64 tick) {
scheduler.Wait(tick);
}
+void BufferCacheRuntime::WaitSubmittedTick(u64 tick) {
+ scheduler.GetMasterSemaphore().Wait(tick);
+}
+
void BufferCacheRuntime::Finish() {
scheduler.Finish();
}
diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h
index ff854445b5..d3295ec6ed 100644
--- a/src/video_core/renderer_vulkan/vk_buffer_cache.h
+++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h
@@ -122,6 +122,9 @@ public:
void Wait(u64 tick);
+ /// Waits for a submitted tick without flushing: safe on threads other than the GPU thread.
+ void WaitSubmittedTick(u64 tick);
+
void Finish();
u64 GetDeviceLocalMemory() const;
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index d3dbba6163..978f8be547 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -922,6 +922,11 @@ void RasterizerVulkan::DispatchCompute() {
return;
}
std::scoped_lock lock{texture_cache.mutex, buffer_cache.mutex};
+ // Command list pages the dispatch writes are copied for the split's front right after it.
+ buffer_cache.Nx2CollectListCopies(true);
+ SCOPE_EXIT {
+ buffer_cache.Nx2CollectListCopies(false);
+ };
if (!pipeline->Configure(*kepler_compute, *gpu_memory, scheduler, buffer_cache,
texture_cache)) {
if (kepler_compute->IsNativeMode()) throw Shader::RuntimeError("Native compute resources unavailable");
@@ -944,6 +949,7 @@ void RasterizerVulkan::DispatchCompute() {
}
cmdbuf.DispatchIndirect(indirect_buffer, indirect_offset);
});
+ SubmitNx2ListCopies();
return;
}
const std::array<u32, 3> dim{qmd.grid_dim_x, qmd.grid_dim_y, qmd.grid_dim_z};
@@ -966,6 +972,7 @@ void RasterizerVulkan::DispatchCompute() {
}
cmdbuf.Dispatch(dim[0], dim[1], dim[2]);
});
+ SubmitNx2ListCopies();
// Log compute dispatch
if (GPU::Logging::IsActive() &&
@@ -1007,6 +1014,29 @@ void Vulkan::RasterizerVulkan::DisableGraphicsUniformBuffer(size_t stage, u32 in
void RasterizerVulkan::FlushAll() {}
+void RasterizerVulkan::SubmitNx2ListCopies() {
+ if (buffer_cache.Nx2CopyListPages(scheduler.CurrentTick())) {
+ scheduler.Flush();
+ }
+}
+
+VideoCore::RasterizerInterface::GpuWrittenList RasterizerVulkan::ReadGpuWrittenCommandList(
+ DAddr addr, std::span<u8> out) {
+ // No cache lock: the front only reads the copies, under their own lock.
+ switch (buffer_cache.Nx2ReadListCopy(addr, out)) {
+ case BufferCache::Nx2ListRead::Copied:
+ return GpuWrittenList::Copied;
+ case BufferCache::Nx2ListRead::Synced:
+ return GpuWrittenList::Synced;
+ default:
+ return GpuWrittenList::Unknown;
+ }
+}
+
+void RasterizerVulkan::NoteGpuWrittenCommandListSynced(DAddr page) {
+ buffer_cache.Nx2NoteListSynced(page);
+}
+
void RasterizerVulkan::FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType which) {
if (addr == 0 || size == 0) {
return;
@@ -1319,6 +1349,8 @@ void RasterizerVulkan::AccelerateInlineToMemory(GPUVAddr address, size_t copy_si
bool uploaded;
{
std::scoped_lock lock{buffer_cache.mutex};
+ // Guest memory has the words now: a command list copy of the page no longer does.
+ buffer_cache.Nx2NoteListHostWrite(*cpu_addr, copy_size);
uploaded = buffer_cache.InlineMemory(*cpu_addr, copy_size, memory);
if (!uploaded && !defer) {
buffer_cache.WriteMemory(*cpu_addr, copy_size);
@@ -1414,7 +1446,13 @@ bool AccelerateDMA::BufferClear(GPUVAddr src_address, u64 amount, u32 value) {
bool AccelerateDMA::BufferCopy(GPUVAddr src_address, GPUVAddr dest_address, u64 amount) {
std::scoped_lock lock{buffer_cache.mutex};
- return buffer_cache.DMACopy(src_address, dest_address, amount);
+ buffer_cache.Nx2CollectListCopies(true);
+ const bool copied = buffer_cache.DMACopy(src_address, dest_address, amount);
+ if (buffer_cache.Nx2CopyListPages(scheduler.CurrentTick())) {
+ scheduler.Flush();
+ }
+ buffer_cache.Nx2CollectListCopies(false);
+ return copied;
}
template <bool IS_IMAGE_UPLOAD>
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h
index fc282fbe8d..dbb6ebdca8 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.h
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.h
@@ -99,6 +99,8 @@ public:
void FlushAll() override;
void FlushRegion(DAddr addr, u64 size,
VideoCommon::CacheType which = VideoCommon::CacheType::All) override;
+ GpuWrittenList ReadGpuWrittenCommandList(DAddr addr, std::span<u8> out) override;
+ void NoteGpuWrittenCommandListSynced(DAddr page) override;
bool MustFlushRegion(DAddr addr, u64 size,
VideoCommon::CacheType which = VideoCommon::CacheType::All) override;
VideoCore::RasterizerDownloadArea GetFlushArea(DAddr addr, u64 size) override;
@@ -165,6 +167,10 @@ private:
void FlushWork();
+ /// Records the copies of the command list pages the work just recorded wrote and submits, so
+ /// the GPU makes them now (BufferCache::Nx2CopyListPages).
+ void SubmitNx2ListCopies();
+
void UpdateDynamicStates();
void HandleTransformFeedback();
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-04 01:57:13 +0000