diff options
| author | Duck912 <350528965+Duck912@users.noreply.github.com> | 2026-11-02 18:56:59 -0500 |
|---|---|---|
| committer | Duck912 <350528965+Duck912@users.noreply.github.com> | 2026-11-02 18:56:59 -0500 |
| commit | 9acd36eb9a291e0631346f5b17d69d0e8eb26cc5 (patch) | |
| tree | 5142f9867928a2e7f07a6b6b79ca00fa5f900425 /src/video_core | |
| parent | 11cebf56014a5015db28179a8acc81cad0caf641 (diff) | |
| parent | 2d81fd5072c060737c18ed4cfc46a04fb8cbb94c (diff) | |
Merge remote-tracking branch 'origin/nx2' into fe-fsr31
# Conflicts:
# src/video_core/renderer_vulkan/blit_image.cpp
# src/video_core/renderer_vulkan/blit_image.h
Diffstat (limited to 'src/video_core')
44 files changed, 607 insertions, 145 deletions
diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index cd176dec27..40369b16ef 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -10,11 +10,13 @@ #include <memory> #include <numeric> #include <stdexcept> +#include <utility> #include <vector> #include <fmt/ranges.h> #include "common/range_sets.inc" #include "video_core/buffer_cache/buffer_cache_base.h" +#include "video_core/buffer_cache/uniform_stream_reuse.h" #include "video_core/guest_memory.h" #include "video_core/host1x/gpu_device_memory_manager.h" #include "video_core/nx2.h" @@ -28,6 +30,9 @@ using Core::DEVICE_PAGESIZE; template <class P> BufferCache<P>::BufferCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, Runtime& runtime_) : runtime{runtime_}, device_memory{device_memory_}, memory_tracker{device_memory} { + // A new runtime has a new scheduler (its ticks restart) and a new stream buffer: whatever the + // stream uniform reuse remembers refers to the old ones. Before the early return below. + VideoCommon::ResetUniformStreamReuse<Runtime>(); // Ensure the first slot is used for the null buffer void(slot_buffers.insert(runtime, NullBufferParams{})); gpu_modified_ranges.Clear(); @@ -180,6 +185,7 @@ void BufferCache<P>::WriteMemory(DAddr device_addr, u64 size) { gpu_modified_ranges.Subtract(device_addr, size); } memory_tracker.MarkRegionAsCpuModified(device_addr, size); + MarkVertexBuffersStale(device_addr, size); } template <class P> @@ -230,8 +236,17 @@ std::optional<VideoCore::RasterizerDownloadArea> BufferCache<P>::GetFlushArea(DA }; area->preemtive = !IsRegionGpuModified(device_addr_start_aligned, device_addr_end_aligned - device_addr_start_aligned); - memory_tracker.MarkRegionAsPreflushable(device_addr_start_aligned, - device_addr_end_aligned - device_addr_start_aligned); + // Mark only data that the GPU wrote. A mark stays for the session and sends later GPU writes + // in the range to a download at each fence. + if (!area->preemtive) { + memory_tracker.MarkRegionAsPreflushable(device_addr_start_aligned, + device_addr_end_aligned - device_addr_start_aligned); + } else { + // The core keeps this area and has no mark. The next GPU write here must arm the range + // again, which ends the area and sets the trap. + read_trapped_ranges.Subtract(device_addr_start_aligned, + device_addr_end_aligned - device_addr_start_aligned); + } return area; } @@ -265,6 +280,7 @@ bool BufferCache<P>::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am } ClearDownload(*cpu_dest_address, amount); + MarkVertexBuffersStale(*cpu_dest_address, amount); BufferId buffer_a; BufferId buffer_b; @@ -408,7 +424,8 @@ void BufferCache<P>::BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr template <class P> void BufferCache<P>::DisableGraphicsUniformBuffer(size_t stage, u32 index) { auto& current = channel_state->uniform_buffers[stage][index]; - if (current.device_addr == 0 && current.size == 0 && !current.buffer_id) [[likely]] { + if (current.device_addr == 0 && current.size == 0 && current.buffer_id == NULL_BUFFER_ID) + [[likely]] { return; } current = NULL_BINDING; @@ -893,7 +910,39 @@ void BufferCache<P>::UpdateVertexBufferSlot(u32 index, const Binding& binding) { } template <class P> +void BufferCache<P>::MarkVertexBuffersStale(DAddr device_addr, u64 size) { + if constexpr (requires { runtime.IsVertexRepacked(u32{}); }) { + u32 mask = enabled_vertex_buffers_mask; + while (mask != 0) { + const u32 index = std::countr_zero(mask); + mask &= mask - 1; + const Binding& binding = v_buffer[index]; + if (device_addr < binding.device_addr + binding.size && + binding.device_addr < device_addr + size) { + stale_vertex_buffers_mask |= 1U << index; + } + } + } +} + +template <class P> +void BufferCache<P>::RefreshRepackedVertexBuffers() { + if constexpr (requires { runtime.IsVertexRepacked(u32{}); }) { + u32 mask = std::exchange(stale_vertex_buffers_mask, 0) & enabled_vertex_buffers_mask; + auto& flags = maxwell3d->dirty.flags; + while (mask != 0) { + const u32 index = std::countr_zero(mask); + mask &= mask - 1; + if (runtime.IsVertexRepacked(index)) { + flags[Dirty::VertexBuffer0 + index] = true; + } + } + } +} + +template <class P> void BufferCache<P>::BindHostVertexBuffers() { + RefreshRepackedVertexBuffers(); #ifdef __ANDROID__ const bool use_optimized_vertex_buffers = Settings::values.use_optimized_vertex_buffers.GetValue(); @@ -1078,8 +1127,20 @@ void BufferCache<P>::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 channel_state->fast_bound_uniform_buffers[stage] |= 1u << binding_index; channel_state->uniform_buffer_binding_sizes[stage][binding_index] = size; // Stream buffer path to avoid stalling on non-Nvidia drivers or Vulkan - const std::span<u8> span = runtime.BindMappedUniformBuffer(stage, binding_index, size); - device_memory.ReadBlockUnsafe(device_addr, span.data(), size); + bool streamed = false; + if constexpr (!IS_OPENGL && VideoCommon::StreamUniformReusable<Runtime>) { + // The same bytes as this slot's previous bind, in the same command buffer: bind that + // stream allocation again instead of allocating and copying (uniform_stream_reuse.h). + // EDEN_NX2_UBOREUSE=0 turns it off. + if (VideoCommon::UniformStreamReuseEnabled()) { + streamed = VideoCommon::BindSmallUniformReusing( + runtime, device_memory, stage, index, binding_index, device_addr, size); + } + } + if (!streamed) { + const std::span<u8> span = runtime.BindMappedUniformBuffer(stage, binding_index, size); + device_memory.ReadBlockUnsafe(device_addr, span.data(), size); + } return; } // Classic cached path @@ -1655,6 +1716,7 @@ void BufferCache<P>::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u3 } } memory_tracker.MarkRegionAsGpuModified(device_addr, size); + MarkVertexBuffersStale(device_addr, size); gpu_modified_ranges.Add(device_addr, size); if (DownloadsAtFence(device_addr, size)) { uncommitted_gpu_modified_ranges.Add(device_addr, size); @@ -2205,6 +2267,7 @@ void BufferCache<P>::InlineMemoryImplementation(DAddr dest_address, size_t copy_ std::span<const u8> inlined_buffer) { ClearDownload(dest_address, copy_size); gpu_modified_ranges.Subtract(dest_address, copy_size); + MarkVertexBuffersStale(dest_address, copy_size); BufferId buffer_id = FindBuffer(dest_address, static_cast<u32>(copy_size), false); auto& buffer = slot_buffers[buffer_id]; diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 1f0c6cf10f..86c05c8f22 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -59,7 +59,8 @@ constexpr u32 NUM_VERTEX_BUFFERS = 32; constexpr u32 NUM_TRANSFORM_FEEDBACK_BUFFERS = 4; constexpr u32 NUM_GRAPHICS_UNIFORM_BUFFERS = 18; constexpr u32 NUM_COMPUTE_UNIFORM_BUFFERS = 8; -constexpr u32 NUM_STORAGE_BUFFERS = 16; +// Native compute shaders bind up to 18 global memory ranges; the enable masks are 32 bits wide. +constexpr u32 NUM_STORAGE_BUFFERS = 32; constexpr u32 NUM_TEXTURE_BUFFERS = 32; constexpr u32 NUM_STAGES = 5; @@ -525,6 +526,13 @@ private: void UpdateVertexBufferSlot(u32 index, const Binding& binding); + /// Flags the vertex buffers overlapping a written range so host copies derived from them are + /// rebuilt. + void MarkVertexBuffersStale(DAddr device_addr, u64 size); + + /// Re-dirties repacked vertex buffers whose source data was written since they were bound. + void RefreshRepackedVertexBuffers(); + void ClearDownload(DAddr base_addr, u64 size); void InlineMemoryImplementation(DAddr dest_address, size_t copy_size, @@ -545,6 +553,7 @@ private: u32 last_index_count = 0; u32 enabled_vertex_buffers_mask = 0; + u32 stale_vertex_buffers_mask = 0; u64 vertex_buffers_serial = 0; std::array<Binding, 32> v_buffer{}; diff --git a/src/video_core/buffer_cache/uniform_stream_reuse.h b/src/video_core/buffer_cache/uniform_stream_reuse.h new file mode 100644 index 0000000000..7ba31c4717 --- /dev/null +++ b/src/video_core/buffer_cache/uniform_stream_reuse.h @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include <array> +#include <concepts> +#include <cstddef> +#include <cstdlib> +#include <cstring> +#include <span> +#include <type_traits> + +#include "common/common_types.h" + +namespace VideoCommon { + +/// What a buffer cache runtime offers for stream uniform reuse. +/// - StreamUniformRef: a small trivially copyable description of one stream allocation (the +/// host buffer, its offset and whatever else the runtime needs to add it to a descriptor +/// payload again, e.g. the device address of the descriptor buffer path). +/// - LastStreamUniformRef(): the allocation made by the latest BindMappedUniformBuffer call. +/// - RebindStreamUniform(ref, size): adds `ref` to the descriptor payload exactly as +/// BindMappedUniformBuffer added the allocation the first time. +template <class R> +concept StreamUniformReusable = requires(R& runtime, u32 size) { + typename R::StreamUniformRef; + requires std::is_trivially_copyable_v<typename R::StreamUniformRef>; + { runtime.LastStreamUniformRef() } -> std::same_as<typename R::StreamUniformRef>; + runtime.RebindStreamUniform(runtime.LastStreamUniformRef(), size); + { runtime.CurrentTick() } -> std::convertible_to<u64>; +}; + +/// EDEN_NX2_UBOREUSE=0 turns the reuse off (every small uniform buffer is copied, as before). +[[nodiscard]] inline bool UniformStreamReuseEnabled() noexcept { + static const bool enabled = [] { + const char* const value = std::getenv("EDEN_NX2_UBOREUSE"); + return value == nullptr || value[0] != '0'; + }(); + return enabled; +} + +template <class Ref> +class UniformStreamReuse { +public: + static constexpr u32 MaxBytes = 4096; // BufferCache's DEFAULT_SKIP_CACHE_SIZE + static constexpr size_t NumStages = 5; + static constexpr size_t NumSlots = 18; // Maxwell constant buffers per stage + + struct Slot { + Ref ref{}; + u64 tick{}; + u32 size{}; // 0: nothing streamed yet + std::array<u8, MaxBytes> bytes{}; + }; + + void Reset() noexcept { + for (auto& stage : slots) { + for (Slot& slot : stage) { + slot.size = 0; + slot.tick = 0; + } + } + } + + std::array<std::array<Slot, NumSlots>, NumStages> slots{}; + std::array<u8, MaxBytes> scratch{}; +}; + +template <class Ref> +[[nodiscard]] inline UniformStreamReuse<Ref>& UniformStreamReuseState() noexcept { + static UniformStreamReuse<Ref> state; + return state; +} + +/// Forgets every slot. Call it when a buffer cache (and so its runtime, scheduler and stream +/// buffer) is created: ticks restart with a new scheduler and the old references are dead. +template <class Runtime> +inline void ResetUniformStreamReuse() noexcept { + if constexpr (StreamUniformReusable<Runtime>) { + UniformStreamReuseState<typename Runtime::StreamUniformRef>().Reset(); + } +} + +/// Binds the small uniform buffer at `device_addr` for (stage, cbuf `index`) through the stream +/// buffer, reusing the slot's previous allocation when the bytes and the command buffer are the +/// same. Returns false (nothing bound) when the buffer does not qualify, so the caller binds it +/// the classic way. +template <class Runtime, class DeviceMemory> +[[nodiscard]] bool BindSmallUniformReusing(Runtime& runtime, DeviceMemory& device_memory, + size_t stage, u32 index, u32 binding_index, + u64 device_addr, u32 size) { + using Reuse = UniformStreamReuse<typename Runtime::StreamUniformRef>; + if (size == 0 || size > Reuse::MaxBytes || stage >= Reuse::NumStages || + index >= Reuse::NumSlots) { + return false; + } + Reuse& reuse = UniformStreamReuseState<typename Runtime::StreamUniformRef>(); + typename Reuse::Slot& slot = reuse.slots[stage][index]; + u8* const scratch = reuse.scratch.data(); + device_memory.ReadBlockUnsafe(device_addr, scratch, size); + + const bool same_bytes = + slot.size == size && std::memcmp(slot.bytes.data(), scratch, size) == 0; + if (same_bytes && slot.tick == runtime.CurrentTick()) { + runtime.RebindStreamUniform(slot.ref, size); + return true; + } + const std::span<u8> span = runtime.BindMappedUniformBuffer(stage, binding_index, size); + std::memcpy(span.data(), scratch, size); + slot.ref = runtime.LastStreamUniformRef(); + // After the allocation: the tick the stream region is tagged with is the one current now. + slot.tick = runtime.CurrentTick(); + if (!same_bytes) { + slot.size = size; + std::memcpy(slot.bytes.data(), scratch, size); + } + return true; +} + +} // namespace VideoCommon diff --git a/src/video_core/control/scheduler.cpp b/src/video_core/control/scheduler.cpp index dc42906752..97d2dcd482 100644 --- a/src/video_core/control/scheduler.cpp +++ b/src/video_core/control/scheduler.cpp @@ -48,6 +48,7 @@ void Scheduler::Push(GPU& gpu, s32 channel, CommandList&& entries) { std::find(pending_channels.begin(), pending_channels.end(), channel_state) == pending_channels.end()) { pending_channels.push_back(channel_state); + any_pending.store(true, std::memory_order_release); } if (!pending_channels.empty()) { ResumePending(gpu); @@ -77,6 +78,7 @@ void Scheduler::ResumePending(GPU& gpu) { std::erase_if(pending_channels, [](const auto& state) { return !state->payload->dma_pusher.HasPendingAcquire(); }); + any_pending.store(!pending_channels.empty(), std::memory_order_release); } void Scheduler::DeclareChannel(std::shared_ptr<ChannelState> new_channel) { diff --git a/src/video_core/control/scheduler.h b/src/video_core/control/scheduler.h index 9a0f1b919f..b1aaf46b60 100644 --- a/src/video_core/control/scheduler.h +++ b/src/video_core/control/scheduler.h @@ -6,6 +6,7 @@ #pragma once +#include <atomic> #include <memory> #include <mutex> #include <vector> @@ -30,6 +31,10 @@ public: [[nodiscard]] bool HasPendingChannels() const { return !pending_channels.empty(); } + /// HasPendingChannels for threads other than the GPU thread. + [[nodiscard]] bool HasPendingChannelsAnyThread() const { + return any_pending.load(std::memory_order_acquire); + } void DeclareChannel(std::shared_ptr<ChannelState> new_channel); @@ -39,6 +44,7 @@ private: ::Common::unordered_map<s32, std::shared_ptr<ChannelState>> channels; std::mutex scheduling_guard; std::vector<std::shared_ptr<ChannelState>> pending_channels; ///< Suspended on an acquire + std::atomic<bool> any_pending{}; }; } // namespace Control diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 6b93bb7f60..6a79e0570e 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -162,6 +162,9 @@ bool DmaPusher::Step() { // read: their words at submit time are an older frame's. const bool submitted = command_resume_offset == 0 && dma_pushbuffer_subindex < command_list.submitted_offsets.size() && + (command_list.snapshot_suspensions & 1) == 0 && + command_list.snapshot_suspensions == + suspensions.load(std::memory_order_acquire) && !IsGpuWrittenCommands(dma_state.dma_get, size_t{header.size} * sizeof(u32)) && !(use_safe && memory_manager.IsMemoryDirty(dma_state.dma_get, diff --git a/src/video_core/dma_pusher.h b/src/video_core/dma_pusher.h index 289786bb9e..feab652bb5 100644 --- a/src/video_core/dma_pusher.h +++ b/src/video_core/dma_pusher.h @@ -7,6 +7,7 @@ #pragma once #include <array> +#include <atomic> #include <condition_variable> #include <optional> #include <span> @@ -121,6 +122,10 @@ struct CommandList final { /// executed instead of the memory, which the game may already be refilling for a later frame. std::vector<CommandHeader> submitted_words; std::vector<u32> submitted_offsets; + /// DmaPusher::suspensions when the words were copied. A list copied during a suspension, or + /// that waited behind one, reads memory instead: games write the commands it runs before they + /// release the acquire. + u64 snapshot_suspensions = 0; }; /// FNV-1a over pushbuffer words (EDEN_NX2_PB_CHECK). @@ -139,6 +144,10 @@ inline u64 HashPushbufferWords(std::span<const u32> words) { /// details on this implementation. class DmaPusher final { public: + /// Counts suspensions on a semaphore acquire and the resumes after them, so it is odd while + /// the channel is suspended (see CommandList::snapshot_suspensions). + std::atomic<u64> suspensions{0}; + explicit DmaPusher(Core::System& system_, MemoryManager& memory_manager_, Control::ChannelState& channel_state_); ~DmaPusher(); diff --git a/src/video_core/engines/draw_manager.cpp b/src/video_core/engines/draw_manager.cpp index 038e5a2e69..564220b71c 100644 --- a/src/video_core/engines/draw_manager.cpp +++ b/src/video_core/engines/draw_manager.cpp @@ -72,11 +72,19 @@ void Maxwell3D::DrawManager::Clear(Maxwell3D& maxwell3d, u32 layer_count) { } } -void Maxwell3D::DrawManager::DrawDeferred(Maxwell3D& maxwell3d) { +void Maxwell3D::DrawManager::DrawDeferred(Maxwell3D& maxwell3d, bool loop_continues) { if (draw_state.draw_mode != DrawMode::Instance || draw_state.instance_count == 0) { + if (!loop_continues) { + draw_state.drawn_instances = 0; + } return; } - DrawEnd(maxwell3d, draw_state.instance_count + 1, true); + // The first draw of a batch also covers the loop's first instance; a batch that continues a + // loop split by a register write draws only the instances after the ones already drawn. + const bool continued = draw_state.drawn_instances != 0; + DrawEnd(maxwell3d, draw_state.instance_count + (continued ? 0 : 1), true); + draw_state.drawn_instances = + loop_continues ? draw_state.drawn_instances + draw_state.instance_count : 0; draw_state.instance_count = 0; } @@ -121,6 +129,7 @@ void Maxwell3D::DrawManager::DrawArrayInstanced(Maxwell3D& maxwell3d, Maxwell3D: draw_state.vertex_buffer.count = vertex_count; if (!subsequent) { draw_state.instance_count = 1; + draw_state.drawn_instances = 0; } draw_state.base_instance = draw_state.instance_count - 1; draw_state.draw_mode = DrawMode::Instance; @@ -220,6 +229,9 @@ void Maxwell3D::DrawManager::DrawEnd(Maxwell3D& maxwell3d, u32 instance_count, b [[fallthrough]]; case DrawMode::General: draw_state.base_instance = maxwell3d.regs.global_base_instance_index; + if (draw_state.draw_mode == DrawMode::Instance && draw_state.drawn_instances != 0) { + draw_state.base_instance += draw_state.drawn_instances + 1; + } draw_state.base_index = maxwell3d.regs.global_base_vertex_index; if (draw_state.draw_indexed) { draw_state.index_buffer = maxwell3d.regs.index_buffer; @@ -301,6 +313,8 @@ void Maxwell3D::DrawManager::UpdateTopology(Maxwell3D& maxwell3d) { } void Maxwell3D::DrawManager::ProcessDraw(Maxwell3D& maxwell3d, bool draw_indexed, u32 instance_count) { + draw_state.draw_element_count = + draw_indexed ? draw_state.index_buffer.count : draw_state.vertex_buffer.count; LOG_TRACE(HW_GPU, "called, topology={}, count={}", draw_state.topology, draw_indexed ? draw_state.index_buffer.count : draw_state.vertex_buffer.count); UpdateTopology(maxwell3d); if (maxwell3d.ShouldExecute()) { @@ -309,6 +323,7 @@ void Maxwell3D::DrawManager::ProcessDraw(Maxwell3D& maxwell3d, bool draw_indexed } void Maxwell3D::DrawManager::ProcessDrawIndirect(Maxwell3D& maxwell3d) { + draw_state.draw_element_count = 0; LOG_TRACE(HW_GPU, "called, topology={}, is_indexed={}, includes_count={}, buffer_size={}, max_draw_count={}", draw_state.topology, indirect_state.is_indexed, indirect_state.include_count, indirect_state.buffer_size, indirect_state.max_draw_counts); UpdateTopology(maxwell3d); if (maxwell3d.ShouldExecute()) { diff --git a/src/video_core/engines/fermi_2d.h b/src/video_core/engines/fermi_2d.h index 7fddf9fdc1..04a12fd351 100644 --- a/src/video_core/engines/fermi_2d.h +++ b/src/video_core/engines/fermi_2d.h @@ -70,6 +70,9 @@ public: ROP = 4, SrcCopyPremult = 5, BlendPremult = 6, + // Not a hardware operation: a copy that decodes sRGB-encoded values to linear, for the + // nx2 upscale fallback standing in for a program that decodes its output. + SrcCopySrgbToLinear = 7, }; enum class MemoryLayout : u32 { diff --git a/src/video_core/engines/kepler_compute.cpp b/src/video_core/engines/kepler_compute.cpp index 2d35b603ae..be6a1d5eac 100644 --- a/src/video_core/engines/kepler_compute.cpp +++ b/src/video_core/engines/kepler_compute.cpp @@ -694,8 +694,32 @@ bool KeplerCompute::UpscaleInsteadOfNativeProgram() { result.addr_lower = static_cast<u32>(tic.Address()); return result; }; + // Title 4's program ends by decoding its result from sRGB to linear (the 0.04045 threshold + // and the 2.4 exponent as half-precision immediates of HSETP2 and HMUL2). The copy of its + // sRGB input left the frame gamma-encoded twice, washed out: decode in the copy when the + // program does. + static std::unordered_map<GPUVAddr, bool> decodes_srgb; + const GPUVAddr program = regs.code_loc.Address(); + const auto [known, inserted] = decodes_srgb.try_emplace(program, false); + if (inserted) { + bool threshold = false, exponent = false; + std::array<u32, 0x400> block{}; + for (u64 offset = 0; offset < 0x20000 && !(threshold && exponent); offset += sizeof(block)) { + if (!memory_manager.IsFullyMappedRange(program + offset, sizeof(block))) break; + memory_manager.ReadBlock(program + offset, block.data(), sizeof(block)); + for (size_t word = 0; word < block.size(); word += 4) { + const u32 opcode = block[word] & 0xfff; + threshold |= opcode == 0x434 && block[word + 1] == 0x292d292d; + exponent |= opcode == 0x832 && block[word + 1] == 0x40cc40cc; + } + } + known->second = threshold && exponent; + LOG_INFO(HW_GPU, "Upscale fallback for program {:#x}: {}", program, + known->second ? "decodes sRGB like the program" : "plain copy"); + } const Fermi2D::Config config{ - .operation = Fermi2D::Operation::SrcCopy, + .operation = known->second ? Fermi2D::Operation::SrcCopySrgbToLinear + : Fermi2D::Operation::SrcCopy, .filter = Fermi2D::Filter::Bilinear, .must_accelerate = true, .dst_x0 = 0, diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 9731f54584..92313988bd 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -342,7 +342,7 @@ void Maxwell3D::ProcessDirtyRegisters(u32 method, u32 argument) { // the state they were begun with: NVN's draw macros reset the base instance (and its copy in // c[0][0x644]) right after the loop, before the macro returns. if (draw_manager.HasDeferredDraw() && !DrawManager::IsInstanceLoopMethod(method)) [[unlikely]] { - draw_manager.DrawDeferred(*this); + draw_manager.DrawDeferred(*this, true); } regs.reg_array[method] = argument; for (auto const& table : dirty.tables) @@ -518,6 +518,10 @@ void Maxwell3D::DumpRecentWrites(u32 method) { } LOG_ERROR(HW_GPU, "Write to {:#x} while macro {:#x} collects parameters ({} so far); last writes:", method, executing_macro, macro_params.size()); + if (!record_write_history) { + LOG_ERROR(HW_GPU, "Register write history disabled; EDEN_NX2_3D_HISTORY=1 enables it"); + return; + } for (size_t i = 0; i < recent_writes.size(); ++i) { const auto& w = recent_writes[(recent_write_index + i) % recent_writes.size()]; if (w.amount != 0) { @@ -537,7 +541,7 @@ void Maxwell3D::CallMultiMethod(Core::System& system, u32 method, const u32* bas } // As in ProcessDirtyRegisters: the paths below write without it. if (draw_manager.HasDeferredDraw() && !DrawManager::IsInstanceLoopMethod(method)) [[unlikely]] { - draw_manager.DrawDeferred(*this); + draw_manager.DrawDeferred(*this, true); } switch (method) { case MAXWELL3D_REG_INDEX(const_buffer.buffer): @@ -818,7 +822,6 @@ struct Maxwell3D::FrontState { Regs::ShadowRamControl control{}; u32 executing_macro{}; std::vector<u32> parameters; - std::vector<u32> fifo; u64 calls{}; /// A method the draw manager handles was sent since the back last ran DrawDeferred: only /// then can it have a deferred instanced draw to issue at the end of a macro. @@ -1095,9 +1098,9 @@ void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split engine.native_jit = std::make_shared<TuringMacroJit>(); } engine.native_jit->Prepare(engine.native_code, engine.native_code_generation); - fs.fifo.assign(fs.parameters.begin(), fs.parameters.end()); + const size_t parameter_count = fs.parameters.size(); const auto result = engine.native_jit->Execute( - start, fs.fifo, std::span<u32>{native_data_ram}, + start, fs.parameters, std::span<u32>{native_data_ram}, [&fs](u32 address) { return address < Regs::NUM_REGS ? fs.regs[address] : 0U; }, [&](u32 address, u32 value) { if (address >= MacroRegistersStart) { @@ -1113,7 +1116,7 @@ void Maxwell3D::FrontRunMacro(Core::System& system, VideoCommon::GpuSplit& split if (!result.success) { engine.native_execution_failed = true; LOG_ERROR(HW_GPU, "Native MME (JIT, split front) failed: start={:#x}, consumed={}/{}, error={}", - start, result.consumed, fs.parameters.size(), result.error); + start, result.consumed, parameter_count, result.error); } } fs.parameters.clear(); @@ -1148,7 +1151,7 @@ void Maxwell3D::FrontDmaRead(VideoCommon::GpuSplit& split, FrontState& fs, u32 m for (u32 i = 0; i < words; ++i) { const u32 word = memory_manager.Read<u32>(source + u64{i} * sizeof(u32)); if (method == 0x158) { - fs.fifo.push_back(word); + fs.parameters.push_back(word); } else if (ram + i < native_data_ram.size()) { native_data_ram[ram + i] = word; } diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index af0e37978e..7646ddbbe3 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3064,11 +3064,14 @@ public: Maxwell3D::Regs::PrimitiveTopology topology{}; DrawMode draw_mode{}; bool draw_indexed{}; + u32 draw_element_count{}; // Zero for indirect draws whose count is not known yet. u32 base_index{}; Maxwell3D::Regs::VertexBuffer vertex_buffer; Maxwell3D::Regs::IndexBuffer index_buffer; u32 base_instance{}; u32 instance_count{}; + /// Deferred instances already drawn by flushes in the middle of the current loop. + u32 drawn_instances{}; std::vector<u8> inline_index_draw_indexes; }; struct DrawTextureState { @@ -3095,7 +3098,8 @@ public: }; void ProcessMethodCall(Maxwell3D& maxwell3d, u32 method, u32 argument); void Clear(Maxwell3D& maxwell3d, u32 layer_count); - void DrawDeferred(Maxwell3D& maxwell3d); + /// Draws the waiting instances. loop_continues is set when the loop goes on afterwards. + void DrawDeferred(Maxwell3D& maxwell3d, bool loop_continues = false); /// Instances after the first wait to be drawn together (DrawDeferred). [[nodiscard]] bool HasDeferredDraw() const noexcept { return draw_state.draw_mode == DrawMode::Instance && draw_state.instance_count != 0; @@ -3349,7 +3353,17 @@ private: }; std::array<RecentWrite, 48> recent_writes{}; size_t recent_write_index = 0; + const bool record_write_history{ +#ifdef NDEBUG + std::getenv("EDEN_NX2_3D_HISTORY") != nullptr +#else + true +#endif + }; void RecordWrite(u32 method, u32 argument, u32 amount, u32 pending) { + if (!record_write_history) { + return; + } recent_writes[recent_write_index++ % recent_writes.size()] = {method, argument, amount, pending}; } void DumpRecentWrites(u32 method); diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 0bb1809aa3..37ac3ec851 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -116,6 +116,7 @@ bool Puller::ResumeAcquire(DmaPusher& dma_pusher) { return false; } pending_acquire.reset(); + dma_pusher.suspensions.fetch_add(1, std::memory_order_release); Nx2SemaphoreWaiters::Get().Remove(this); return true; } @@ -257,6 +258,7 @@ void Puller::ProcessSemaphoreTriggerMethod(DmaPusher& dma_pusher) { dma_pusher.rasterizer->ReleaseFences(); if (!waiters.Latched(this) && !acquire.Matches(ReadSemaphoreWord(dma_pusher, acquire.address))) { pending_acquire = acquire; + dma_pusher.suspensions.fetch_add(1, std::memory_order_release); VideoCommon::Nx2Profile::Add(VideoCommon::Nx2Profile::Counter::AcquireWaits, 1); } else { waiters.Remove(this); diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index 53f8f063f7..6dbabcb43a 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -210,6 +210,10 @@ struct GPU::Impl { gpu_thread.Run(std::move(func), is_async); } + void MapInOrder(std::function<void()>&& func) { + gpu_thread.Run(std::move(func), is_async, true); + } + /// Notify rasterizer that any caches of the specified region should be flushed to Switch memory void FlushRegion(DAddr addr, u64 size) { gpu_thread.FlushRegion(addr, size, is_async); @@ -228,7 +232,9 @@ struct GPU::Impl { raster_area.start_address, raster_area.end_address, reads.load(std::memory_order_relaxed)); } - const u64 fence = RequestSyncOperation([this, &raster_area]() { + const u64 fence = RequestSyncOperation([this, raster_area]() { + // Apply collected guest writes before downloading GPU data over guest memory. + InvalidateGPUCache(); renderer->ReadRasterizer()->FlushRegion(raster_area.start_address, raster_area.end_address - raster_area.start_address); }); gpu_thread.TickGPU(is_async); @@ -505,6 +511,10 @@ bool GPU::IsAsync() const { return impl->is_async; } +bool GPU::HasSuspendedChannels() const { + return impl->scheduler.HasPendingChannelsAnyThread(); +} + bool GPU::UseNvdec() const { return impl->use_nvdec; } @@ -549,6 +559,10 @@ void GPU::RunInOrder(std::function<void()>&& func) { impl->RunInOrder(std::move(func)); } +void GPU::MapInOrder(std::function<void()>&& func) { + impl->MapInOrder(std::move(func)); +} + void GPU::InvalidateRegion(DAddr addr, u64 size, bool preserve_gpu_writes) { impl->InvalidateRegion(addr, size, preserve_gpu_writes); } diff --git a/src/video_core/gpu.h b/src/video_core/gpu.h index 3a6546b41b..d37dfa5990 100644 --- a/src/video_core/gpu.h +++ b/src/video_core/gpu.h @@ -217,6 +217,9 @@ public: [[nodiscard]] bool IsAsync() const; + /// Whether a channel is suspended on a semaphore acquire (callable from any thread). + [[nodiscard]] bool HasSuspendedChannels() const; + [[nodiscard]] bool UseNvdec() const; void RendererFrameEndNotify(); @@ -254,6 +257,10 @@ public: /// Runs func on the GPU thread after the command lists submitted so far. void RunInOrder(std::function<void()>&& func); + /// RunInOrder for GPU memory maps: also done before the GPU thread reads the command lists + /// submitted later (with the GPU split, on its front rather than its back). + void MapInOrder(std::function<void()>&& func); + /// Notify rasterizer that any caches of the specified region should be invalidated void InvalidateRegion(DAddr addr, u64 size, bool preserve_gpu_writes = false); diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index 55200b0647..3af8cb65a8 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -99,7 +99,12 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten } else if (std::holds_alternative<GPUTickCommand>(next.data)) { split.Defer([gpu = &system.GPU()] { gpu->TickWork(); }); } else if (auto* run = std::get_if<RunCommand>(&next.data)) { - split.DeferFunction(std::move(run->func)); + if (run->before_later_reads) { + split.Drain(); + run->func(); + } else { + split.DeferFunction(std::move(run->func)); + } } else if (const auto* flush = std::get_if<FlushRegionCommand>(&next.data)) { split.Defer([rasterizer_ptr, addr = flush->addr, size = flush->size] { rasterizer_ptr->FlushRegion(addr, size); @@ -155,8 +160,8 @@ void ThreadManager::TickGPU(bool is_async) { PushCommand(GPUTickCommand(), false, is_async); } -void ThreadManager::Run(std::function<void()>&& func, bool is_async) { - PushCommand(RunCommand{std::move(func)}, false, is_async); +void ThreadManager::Run(std::function<void()>&& func, bool is_async, bool before_later_reads) { + PushCommand(RunCommand{std::move(func), before_later_reads}, false, is_async); } void ThreadManager::InvalidateRegion(DAddr addr, u64 size) { diff --git a/src/video_core/gpu_thread.h b/src/video_core/gpu_thread.h index 796c0d89f6..927453dab0 100644 --- a/src/video_core/gpu_thread.h +++ b/src/video_core/gpu_thread.h @@ -80,6 +80,9 @@ struct GPUTickCommand final {}; /// Command to run a function on the GPU thread after the commands queued before it struct RunCommand final { std::function<void()> func; + /// With the GPU split, run on its front (after draining the back) instead of deferring to the + /// back, so command lists read after this one see what func did (GMMU maps). + bool before_later_reads{}; }; using CommandData = @@ -132,7 +135,7 @@ public: void TickGPU(bool is_async); /// Runs func on the GPU thread once the commands queued before it have been processed. - void Run(std::function<void()>&& func, bool is_async); + void Run(std::function<void()>&& func, bool is_async, bool before_later_reads = false); void NotifyShutdown(); diff --git a/src/video_core/host_shaders/CMakeLists.txt b/src/video_core/host_shaders/CMakeLists.txt index 6628d4cce3..7fbf9a80be 100644 --- a/src/video_core/host_shaders/CMakeLists.txt +++ b/src/video_core/host_shaders/CMakeLists.txt @@ -18,6 +18,7 @@ set(SHADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/astc_decoder.comp ${CMAKE_CURRENT_SOURCE_DIR}/bc_encoder.comp ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_float.frag + ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_srgb_to_linear.frag ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_2d.comp ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_msaa.frag ${CMAKE_CURRENT_SOURCE_DIR}/blit_depth.frag diff --git a/src/video_core/host_shaders/blit_color_srgb_to_linear.frag b/src/video_core/host_shaders/blit_color_srgb_to_linear.frag new file mode 100644 index 0000000000..1a38cb52a1 --- /dev/null +++ b/src/video_core/host_shaders/blit_color_srgb_to_linear.frag @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#version 450 + +layout(binding = 0) uniform sampler2D tex; + +layout(location = 0) in vec2 texcoord; +layout(location = 0) out vec4 color; + +// Copy of an sRGB-encoded source into a linear destination: the piecewise sRGB decode on the +// color channels, alpha unchanged. +void main() { + const vec4 value = textureLod(tex, texcoord, 0); + const vec3 low = value.rgb / 12.92; + const vec3 high = pow((value.rgb + 0.055) / 1.055, vec3(2.4)); + color = vec4(mix(high, low, lessThan(value.rgb, vec3(0.04045))), value.a); +} diff --git a/src/video_core/renderer_opengl/gl_compute_pipeline.cpp b/src/video_core/renderer_opengl/gl_compute_pipeline.cpp index eb134e86b5..5fc10cfce2 100644 --- a/src/video_core/renderer_opengl/gl_compute_pipeline.cpp +++ b/src/video_core/renderer_opengl/gl_compute_pipeline.cpp @@ -106,19 +106,18 @@ void ComputePipeline::Configure() { const auto& cbufs{qmd.const_buffer_config}; const bool via_header_index{qmd.linked_tsc != 0}; const auto read_handle{[&](const auto& desc, u32 index) { - if constexpr (std::is_same_v<decltype(desc), const Shader::TextureDescriptor&>) { - if (desc.handle_program != ~0U) { - // Read by the shader from a storage buffer: computed for this dispatch. - return TexturePair(VideoCommon::EvaluateTextureHandle( - info, desc.handle_program, *gpu_memory, - [&](u32 cbuf) -> GPUVAddr { - return cbuf < cbufs.size() && - ((qmd.const_buffer_enable_mask >> cbuf) & 1) != 0 - ? cbufs[cbuf].Address() - : GPUVAddr{}; - }), - via_header_index); - } + if (desc.handle_program != ~0U) { + // Read by the shader from a storage buffer: computed for this dispatch. + return TexturePair(VideoCommon::EvaluateTextureHandle( + info, desc.handle_program, *gpu_memory, + [&](u32 cbuf) -> GPUVAddr { + return cbuf < cbufs.size() && + ((qmd.const_buffer_enable_mask >> cbuf) & + 1) != 0 + ? cbufs[cbuf].Address() + : GPUVAddr{}; + }), + via_header_index); } ASSERT(((qmd.const_buffer_enable_mask >> desc.cbuf_index) & 1) != 0); const u32 index_offset{index << desc.size_shift}; diff --git a/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp b/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp index 81950279d5..4e02f0569b 100644 --- a/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp +++ b/src/video_core/renderer_opengl/gl_graphics_pipeline.cpp @@ -309,18 +309,16 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { } const auto& cbufs{maxwell3d->state.shader_stages[stage].const_buffers}; const auto read_handle{[&](const auto& desc, u32 index) { - if constexpr (std::is_same_v<decltype(desc), const Shader::TextureDescriptor&>) { - if (desc.handle_program != ~0U) { - // Read by the shader from a storage buffer: computed for this draw. - return TexturePair(VideoCommon::EvaluateTextureHandle( - info, desc.handle_program, *gpu_memory, - [&](u32 cbuf) -> GPUVAddr { - return cbuf < cbufs.size() && cbufs[cbuf].enabled - ? cbufs[cbuf].address - : GPUVAddr{}; - }), - via_header_index); - } + if (desc.handle_program != ~0U) { + // Read by the shader from a storage buffer: computed for this draw. + return TexturePair(VideoCommon::EvaluateTextureHandle( + info, desc.handle_program, *gpu_memory, + [&](u32 cbuf) -> GPUVAddr { + return cbuf < cbufs.size() && cbufs[cbuf].enabled + ? cbufs[cbuf].address + : GPUVAddr{}; + }), + via_header_index); } ASSERT(cbufs[desc.cbuf_index].enabled); const u32 index_offset{index << desc.size_shift}; diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index ac750ab747..fd27fb1805 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -54,7 +54,7 @@ using VideoCommon::LoadPipelines; using VideoCommon::SerializePipeline; using Context = ShaderContext::Context; -constexpr u32 CACHE_VERSION = 15; +constexpr u32 CACHE_VERSION = 16; template <typename Container> auto MakeSpan(Container& container) { diff --git a/src/video_core/renderer_vulkan/blit_image.cpp b/src/video_core/renderer_vulkan/blit_image.cpp index 2afc16b379..0628da1d0f 100644 --- a/src/video_core/renderer_vulkan/blit_image.cpp +++ b/src/video_core/renderer_vulkan/blit_image.cpp @@ -12,6 +12,7 @@ #include "common/div_ceil.h" #include "common/settings.h" #include "video_core/host_shaders/blit_color_float_frag_spv.h" +#include "video_core/host_shaders/blit_color_srgb_to_linear_frag_spv.h" #include "video_core/host_shaders/blit_color_msaa_frag_spv.h" #include "video_core/host_shaders/blit_depth_frag_spv.h" #include "video_core/host_shaders/blit_depth_msaa_frag_spv.h" @@ -708,6 +709,7 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, blit_color_to_color_frag(BuildShader(device, BLIT_COLOR_FLOAT_FRAG_SPV)), temporal_upscale_frag(BuildShader(device, TEMPORAL_UPSCALE_FRAG_SPV)), temporal_sharpen_frag(BuildShader(device, TEMPORAL_SHARPEN_FRAG_SPV)), + blit_color_srgb_to_linear_frag(BuildShader(device, BLIT_COLOR_SRGB_TO_LINEAR_FRAG_SPV)), blit_color_msaa_frag(BuildShader(device, BLIT_COLOR_MSAA_FRAG_SPV)), blit_depth_stencil_frag(device.IsExtShaderStencilExportSupported() ? BuildShader(device, VULKAN_BLIT_DEPTH_STENCIL_FRAG_SPV) @@ -1403,7 +1405,10 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe } blit_color_keys.push_back(key); - const std::array stages = MakeStages(*full_screen_vert, *blit_color_to_color_frag); + const bool srgb_to_linear = + key.operation == Tegra::Engines::Fermi2D::Operation::SrcCopySrgbToLinear; + const std::array stages = MakeStages( + *full_screen_vert, srgb_to_linear ? *blit_color_srgb_to_linear_frag : *blit_color_to_color_frag); const VkPipelineColorBlendAttachmentState blend_attachment{ .blendEnable = VK_FALSE, .srcColorBlendFactor = VK_BLEND_FACTOR_ZERO, diff --git a/src/video_core/renderer_vulkan/blit_image.h b/src/video_core/renderer_vulkan/blit_image.h index 6c3685a0a6..a1c802848b 100644 --- a/src/video_core/renderer_vulkan/blit_image.h +++ b/src/video_core/renderer_vulkan/blit_image.h @@ -242,6 +242,7 @@ private: vk::ShaderModule blit_color_to_color_frag; vk::ShaderModule temporal_upscale_frag; vk::ShaderModule temporal_sharpen_frag; + vk::ShaderModule blit_color_srgb_to_linear_frag; vk::ShaderModule blit_color_msaa_frag; vk::ShaderModule blit_depth_stencil_frag; vk::ShaderModule blit_depth_frag; diff --git a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp index d1f78c0034..76a564aae9 100644 --- a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp +++ b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp @@ -269,7 +269,7 @@ void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFe if (xfb_enabled) { RefreshXfbState(xfb_state, regs); } - if (!dynamic_vertex_input && ExperimentalStaticVertexRepackEnabled()) { + if (!dynamic_vertex_input && features.has_static_vertex_repack) { const u32 repack_mask = VertexRepackMask(regs); // Refresh from guest values even when only the stride changed: offsets must not accumulate. for (size_t index = 0; index < attributes.size(); ++index) { diff --git a/src/video_core/renderer_vulkan/fixed_pipeline_state.h b/src/video_core/renderer_vulkan/fixed_pipeline_state.h index 36a4b4b7a2..1fe5cd75f3 100644 --- a/src/video_core/renderer_vulkan/fixed_pipeline_state.h +++ b/src/video_core/renderer_vulkan/fixed_pipeline_state.h @@ -33,6 +33,7 @@ struct DynamicFeatures { bool has_dynamic_state3_logic_op_enable; bool has_dynamic_state3_line_stipple_enable; bool has_dynamic_vertex_input; + bool has_static_vertex_repack; bool has_color_write_enable; bool has_provoking_vertex; bool has_provoking_vertex_first_mode; diff --git a/src/video_core/renderer_vulkan/vertex_repack.h b/src/video_core/renderer_vulkan/vertex_repack.h index b0849c25c7..493eb41bd1 100644 --- a/src/video_core/renderer_vulkan/vertex_repack.h +++ b/src/video_core/renderer_vulkan/vertex_repack.h @@ -2,6 +2,7 @@ #pragma once +#include <algorithm> #include <cstdlib> #include <cstring> #include <limits> @@ -14,12 +15,21 @@ constexpr bool VertexRepackInputSupported(bool dynamic_input, bool experimental_ return dynamic_input || experimental_static; } -inline bool ExperimentalStaticVertexRepackEnabled() { - static const bool enabled = [] { - const char* value = std::getenv("EGLANTINE_EXPERIMENTAL_STATIC_VERTEX_REPACK"); - return value && std::strcmp(value, "1") == 0; - }(); - return enabled; +// Pipelines without dynamic vertex input repack when the device cannot fetch the 69-byte stride. +// EGLANTINE_EXPERIMENTAL_STATIC_VERTEX_REPACK=1 forces the repack on, =0 forces it off. +inline bool StaticVertexRepackSelected(const char* override_value, u32 stride_alignment) { + if (override_value && std::strcmp(override_value, "1") == 0) { + return true; + } + if (override_value && std::strcmp(override_value, "0") == 0) { + return false; + } + return stride_alignment > 1 && 69 % stride_alignment != 0; +} + +inline bool StaticVertexRepackEnabled(u32 stride_alignment) { + return StaticVertexRepackSelected(std::getenv("EGLANTINE_EXPERIMENTAL_STATIC_VERTEX_REPACK"), + stride_alignment); } struct VertexRepackParameters { @@ -31,8 +41,11 @@ struct VertexRepackParameters { u32 dispatch_groups; }; -inline std::optional<VertexRepackParameters> MakeVertexRepackParameters( - u32 offset, u32 size, u64 alignment, u64 max_storage_range, u32 max_dispatch_groups) { +inline std::optional<VertexRepackParameters> MakeVertexRepackParameters(u32 offset, u32 size, + u64 alignment, + u64 max_storage_range, + u32 max_dispatch_groups, + u64 buffer_size) { if (size == 0 || alignment == 0) { return std::nullopt; } @@ -40,9 +53,11 @@ inline std::optional<VertexRepackParameters> MakeVertexRepackParameters( const u64 output_size = vertices * 72; const u64 base = u64{offset} / alignment * alignment; const u64 bias = offset - base; - const u64 range = (bias + size + 3) / 4 * 4; + // The word-rounded range must not run past the end of the source buffer + const u64 available = buffer_size > base ? buffer_size - base : 0; + const u64 range = (std::min)((bias + size + 3) / 4 * 4, available); const u64 groups = (vertices * 18 + 127) / 128; - if (output_size > std::numeric_limits<u32>::max() || + if (range < bias + size || output_size > std::numeric_limits<u32>::max() || range > std::numeric_limits<u32>::max() || output_size > max_storage_range || range > max_storage_range || groups > max_dispatch_groups) { return std::nullopt; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 2449265509..43cb7d9414 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -365,6 +365,10 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m LOG_INFO(Render_Vulkan, "69-byte vertex layout repacking {} ({})", enable_vertex_repack ? "enabled" : "disabled", "portable Vulkan layout fallback"); + static_vertex_repack = + StaticVertexRepackEnabled(device.GetMinVertexInputBindingStrideAlignment()); + LOG_INFO(Render_Vulkan, "Vertex repacking without dynamic vertex input {}", + static_vertex_repack ? "enabled" : "disabled"); if (enable_vertex_repack) { vertex_repack_pass = std::make_unique<VertexRepackPass>( device, scheduler, descriptor_pool, staging_pool, compute_pass_descriptor_queue); @@ -387,14 +391,13 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m u32 BufferCacheRuntime::SetVertexRepackLayout(const Tegra::Engines::Maxwell3D::Regs& regs, bool dynamic_input) { const u32 old_mask = vertex_repack_mask; - const bool input_supported = VertexRepackInputSupported( - dynamic_input, ExperimentalStaticVertexRepackEnabled()); + const bool input_supported = VertexRepackInputSupported(dynamic_input, static_vertex_repack); vertex_repack_mask = vertex_repack_pass && input_supported ? VertexRepackMask(regs) : 0; if (vertex_repack_mask != old_mask) { LOG_DEBUG(Render_Vulkan, "Vertex repack selection: mask={:#x}, dynamic_input={}", vertex_repack_mask, dynamic_input); } - return old_mask | vertex_repack_mask; + return old_mask ^ vertex_repack_mask; } StagingBufferRef BufferCacheRuntime::UploadStagingBuffer(size_t size) { @@ -657,7 +660,8 @@ void BufferCacheRuntime::BindVertexBuffer(u32 index, VkBuffer buffer, u32 offset return; } if (buffer != VK_NULL_HANDLE && size != 0 && IsVertexRepacked(index)) { - const auto repacked = vertex_repack_pass->Assemble(buffer, offset, size); + const auto repacked = + vertex_repack_pass->Assemble(buffer, offset, size, u64{offset} + size); if (!repacked) { return; } @@ -702,7 +706,8 @@ void BufferCacheRuntime::BindVertexBuffers(VideoCommon::HostBindings<Buffer>& bi if (handle != VK_NULL_HANDLE && IsVertexRepacked(binding) && bindings.strides[i] == 69 && bindings.sizes[i] > 0) { const auto repacked = vertex_repack_pass->Assemble( - handle, static_cast<u32>(bindings.offsets[i]), static_cast<u32>(bindings.sizes[i])); + handle, static_cast<u32>(bindings.offsets[i]), static_cast<u32>(bindings.sizes[i]), + bindings.buffers[i]->SizeBytes()); if (!repacked) { continue; } @@ -712,7 +717,7 @@ void BufferCacheRuntime::BindVertexBuffers(VideoCommon::HostBindings<Buffer>& bi bindings.strides[i] = 72; static bool reported = false; if (!reported) { - LOG_WARNING(Render_Vulkan, "Experimental GPU vertex repacking active: 69 -> 72 bytes"); + LOG_INFO(Render_Vulkan, "GPU vertex repacking active: 69 -> 72 bytes"); reported = true; } } diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index d3295ec6ed..b9a0225094 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -110,6 +110,7 @@ public: void TickFrame(Common::SlotVector<Buffer>& slot_buffers) noexcept; + /// Returns the mask of bindings whose repacked state changed. u32 SetVertexRepackLayout(const Tegra::Engines::Maxwell3D::Regs& regs, bool dynamic_input); bool IsVertexRepacked(u32 binding) const { @@ -168,15 +169,40 @@ public: void BindTransformFeedbackBuffers(VideoCommon::HostBindings<Buffer>& bindings); + /// This and the two members below are the hooks of VideoCommon::StreamUniformReusable + /// (buffer_cache/uniform_stream_reuse.h). + struct StreamUniformRef { + VkBuffer buffer; + VkDeviceAddress device_address; // base of `buffer`, 0 without buffer device address + u32 offset; + }; + std::span<u8> BindMappedUniformBuffer([[maybe_unused]] size_t stage, [[maybe_unused]] u32 binding_index, u32 size) { const StagingBufferRef ref = staging_pool.Request(size, MemoryUsage::Upload); - guest_descriptor_queue.AddBuffer(ref.buffer, ref.device_address, - static_cast<u32>(ref.offset), size); + last_stream_uniform = StreamUniformRef{ + .buffer = ref.buffer, + .device_address = ref.device_address, + .offset = static_cast<u32>(ref.offset), + }; + RebindStreamUniform(last_stream_uniform, size); return ref.mapped_span; } + /// The allocation of the latest BindMappedUniformBuffer call. + [[nodiscard]] StreamUniformRef LastStreamUniformRef() const noexcept { + return last_stream_uniform; + } + + /// Adds `ref` to the descriptor payload exactly as BindMappedUniformBuffer did when it made + /// the allocation (it is the very same call). The caller guarantees the allocation is still + /// live: made on the scheduler tick that is current now, so the staging pool has not recycled + /// it and the GPU has not even started on it. + void RebindStreamUniform(const StreamUniformRef& ref, u32 size) { + guest_descriptor_queue.AddBuffer(ref.buffer, ref.device_address, ref.offset, size); + } + [[nodiscard]] VkDeviceSize SparseAlignmentFor(bool sparse_compatible) const noexcept { if (!sparse_compatible || !multi_range_buffers.use_sparse) { return 0; @@ -270,12 +296,15 @@ private: std::unique_ptr<Uint8Pass> uint8_pass; std::unique_ptr<VertexRepackPass> vertex_repack_pass; u32 vertex_repack_mask{}; + bool static_vertex_repack{}; QuadIndexedPass quad_index_pass; MultiRangeBufferCache multi_range_buffers; boost::container::small_vector<MultiRangeSource, 16> multi_range_sources; VkDeviceSize multi_range_total{}; + StreamUniformRef last_stream_uniform{}; + bool limit_dynamic_storage_buffers = false; u32 max_dynamic_storage_buffers = (std::numeric_limits<u32>::max)(); }; diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp index cb8dcde42f..5902b7d8f5 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp @@ -335,10 +335,11 @@ VertexRepackPass::VertexRepackPass(const Device& device_, Scheduler& scheduler_, scheduler{scheduler_}, staging_pool{staging_pool_}, descriptor_queue{descriptor_queue_} {} std::optional<std::pair<VkBuffer, VkDeviceSize>> VertexRepackPass::Assemble(VkBuffer source, - u32 offset, u32 size) { + u32 offset, u32 size, + u64 buffer_size) { const auto parameters = MakeVertexRepackParameters( offset, size, device.GetStorageBufferAlignment(), device.GetMaxStorageBufferRange(), - device.GetMaxComputeWorkGroupCount()[0]); + device.GetMaxComputeWorkGroupCount()[0], buffer_size); if (!parameters) { LOG_ERROR(Render_Vulkan, "Vertex repack exceeds device limits: offset={}, size={}", offset, size); return std::nullopt; diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.h b/src/video_core/renderer_vulkan/vk_compute_pass.h index 27fffbbbb7..9687bfff91 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.h +++ b/src/video_core/renderer_vulkan/vk_compute_pass.h @@ -60,9 +60,10 @@ class VertexRepackPass final : public ComputePass { public: VertexRepackPass(const Device& device, Scheduler& scheduler, DescriptorPool& descriptor_pool, StagingBufferPool& staging_pool, ComputePassDescriptorQueue& descriptor_queue); - /// Returns nothing when the range exceeds the device limits; the binding is then left as is. - std::optional<std::pair<VkBuffer, VkDeviceSize>> Assemble(VkBuffer source, u32 offset, - u32 size); + /// Returns nothing when the range exceeds the device limits or the source buffer; the binding + /// is then left as is. + std::optional<std::pair<VkBuffer, VkDeviceSize>> Assemble(VkBuffer source, u32 offset, u32 size, + u64 buffer_size); private: Scheduler& scheduler; diff --git a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp index 1cfe391ba6..3a3861930c 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp @@ -176,19 +176,18 @@ bool ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute, const auto& cbufs{qmd.const_buffer_config}; const bool via_header_index{qmd.linked_tsc != 0}; const auto read_handle{[&](const auto& desc, u32 index) { - if constexpr (std::is_same_v<decltype(desc), const Shader::TextureDescriptor&>) { - if (desc.handle_program != ~0U) { - // Read by the shader from a storage buffer: computed for this dispatch. - return TexturePair(VideoCommon::EvaluateTextureHandle( - info, desc.handle_program, gpu_memory, - [&](u32 cbuf) -> GPUVAddr { - return cbuf < cbufs.size() && - ((qmd.const_buffer_enable_mask >> cbuf) & 1) != 0 - ? cbufs[cbuf].Address() - : GPUVAddr{}; - }), - via_header_index); - } + if (desc.handle_program != ~0U) { + // Read by the shader from a storage buffer: computed for this dispatch. + return TexturePair(VideoCommon::EvaluateTextureHandle( + info, desc.handle_program, gpu_memory, + [&](u32 cbuf) -> GPUVAddr { + return cbuf < cbufs.size() && + ((qmd.const_buffer_enable_mask >> cbuf) & + 1) != 0 + ? cbufs[cbuf].Address() + : GPUVAddr{}; + }), + via_header_index); } ASSERT(((qmd.const_buffer_enable_mask >> desc.cbuf_index) & 1) != 0); const u32 index_offset{index << desc.size_shift}; diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp index fa273f6ef9..4bd2e2735e 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp @@ -400,18 +400,16 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { } const auto& cbufs{maxwell3d->state.shader_stages[stage].const_buffers}; const auto read_handle{[&](const auto& desc, u32 index) { - if constexpr (std::is_same_v<decltype(desc), const Shader::TextureDescriptor&>) { - if (desc.handle_program != ~0U) { - // Read by the shader from a storage buffer: computed for this draw. - return TexturePair(VideoCommon::EvaluateTextureHandle( - info, desc.handle_program, *gpu_memory, - [&](u32 cbuf) -> GPUVAddr { - return cbuf < cbufs.size() && cbufs[cbuf].enabled - ? cbufs[cbuf].address - : GPUVAddr{}; - }), - via_header_index); - } + if (desc.handle_program != ~0U) { + // Read by the shader from a storage buffer: computed for this draw. + return TexturePair(VideoCommon::EvaluateTextureHandle( + info, desc.handle_program, *gpu_memory, + [&](u32 cbuf) -> GPUVAddr { + return cbuf < cbufs.size() && cbufs[cbuf].enabled + ? cbufs[cbuf].address + : GPUVAddr{}; + }), + via_header_index); } ASSERT(cbufs[desc.cbuf_index].enabled); const u32 index_offset{index << desc.size_shift}; diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index ff8e84b019..b651bfc9ba 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -34,6 +34,7 @@ #include "video_core/renderer_vulkan/maxwell_to_vk.h" #include "video_core/renderer_vulkan/pipeline_helper.h" #include "video_core/renderer_vulkan/pipeline_statistics.h" +#include "video_core/renderer_vulkan/vertex_repack.h" #include "video_core/renderer_vulkan/vk_compute_pipeline.h" #include "video_core/renderer_vulkan/vk_descriptor_pool.h" #include "video_core/renderer_vulkan/vk_pipeline_cache.h" @@ -65,7 +66,7 @@ using VideoCommon::FileEnvironment; using VideoCommon::GenericEnvironment; using VideoCommon::GraphicsEnvironment; -constexpr u32 CACHE_VERSION = 18; +constexpr u32 CACHE_VERSION = 19; constexpr size_t VULKAN_CACHE_FLUSH_PIPELINES = 128; constexpr size_t VULKAN_CACHE_FLUSH_MIN_SECONDS = 30; constexpr std::array<char, 8> VULKAN_CACHE_MAGIC_NUMBER{'y', 'u', 'z', 'u', 'v', 'k', 'c', 'h'}; @@ -583,6 +584,8 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, dynamic_features.has_dynamic_vertex_input = device.IsExtVertexInputDynamicStateSupported() && Settings::values.vertex_input_dynamic_state.GetValue(); + dynamic_features.has_static_vertex_repack = + StaticVertexRepackEnabled(device.GetMinVertexInputBindingStrideAlignment()); dynamic_features.has_provoking_vertex = device.IsExtProvokingVertexSupported(); dynamic_features.has_provoking_vertex_first_mode = @@ -600,9 +603,17 @@ constexpr u32 UsedPipelinesMagic = 0x31505545; // "EUP1" // Pipelines the last two sessions used are built at boot. Each holds ~0.45 MB of driver memory: // title 1 built 4,800 from eight sessions, 1.5 GB more than building them on first use (but that // stalled frames 4x as often), while one session of its title screen uses ~1,500. +// MoltenVK keeps the previous eight sessions: a pipeline built while playing goes through +// SPIR-V -> MSL -> Metal, far more than the 2-3 ms the shorter window assumes, so each one +// built at boot is a stutter avoided. constexpr u8 UsedPipelinesMaxAge = 2; +constexpr u8 UsedPipelinesMaxAgeMoltenVK = 8; -std::unordered_map<u64, u8> LoadUsedPipelines(const std::filesystem::path& path) { +u8 MaxUsedPipelineAge(const Device& device) { + return device.IsMoltenVK() ? UsedPipelinesMaxAgeMoltenVK : UsedPipelinesMaxAge; +} + +std::unordered_map<u64, u8> LoadUsedPipelines(const std::filesystem::path& path, u8 max_age) { std::unordered_map<u64, u8> used; std::ifstream file{path, std::ios::binary}; u32 magic{}, count{}; @@ -618,7 +629,7 @@ std::unordered_map<u64, u8> LoadUsedPipelines(const std::filesystem::path& path) !file.read(reinterpret_cast<char*>(&age), sizeof(age))) { break; } - if (age < UsedPipelinesMaxAge) { // lists saved with a longer window + if (age < max_age) { // lists saved with a longer window used.emplace(hash, age); } } @@ -669,7 +680,7 @@ void PipelineCache::SaveUsedPipelines() { next.emplace(hash, u8{0}); } for (const auto& [hash, age] : used_pipelines) { - if (age + 1 < UsedPipelinesMaxAge) { + if (age + 1 < MaxUsedPipelineAge(device)) { next.try_emplace(hash, static_cast<u8>(age + 1)); } } @@ -744,7 +755,7 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading } pipeline_cache_filename = base_dir / "vulkan.bin"; used_pipelines_filename = base_dir / "vulkan_used.bin"; - used_pipelines = LoadUsedPipelines(used_pipelines_filename); + used_pipelines = LoadUsedPipelines(used_pipelines_filename, MaxUsedPipelineAge(device)); used_pipelines_saved = std::chrono::steady_clock::now(); size_t skipped_pipelines = 0; size_t queued_graphics = 0; @@ -1004,6 +1015,11 @@ void PipelineCache::TickFrame() { } void PipelineCache::EvictUnusedPipelines() { + // Building a pipeline again on MoltenVK means a Metal compile mid-game, far dearer than the + // 2-3 ms below, so none are evicted there. + if (device.IsMoltenVK()) { + return; + } // Each built pipeline holds ~0.45 MB of driver memory and none was ever destroyed, so it grew // with every place a session went (title 1: 1,500 on its title screen alone). One unused for // this long is destroyed; it takes 2-3 ms to build again from the driver's pipeline cache. @@ -1056,7 +1072,7 @@ GraphicsPipeline* PipelineCache::BuiltPipeline(GraphicsPipeline* pipeline) const // Usually these shaders are only used once for building textures so we can assume they // can't be built async const auto& draw_state = maxwell3d->draw_manager.draw_state; - if (draw_state.index_buffer.count <= 6 || draw_state.vertex_buffer.count <= 6) { + if (draw_state.draw_element_count <= 6) { return pipeline; } return nullptr; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 6b22c01c43..0e5124dba6 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -1127,12 +1127,10 @@ void RasterizerVulkan::InnerInvalidation(std::span<const std::pair<DAddr, std::s buffer_cache.WriteMemory(addr, size); } } - { - for (const auto& [addr, size] : sequences) { - query_cache.InvalidateRegion(addr, size); - pipeline_cache.InvalidateRegion(addr, size); - } + for (const auto& [addr, size] : sequences) { + query_cache.InvalidateRegion(addr, size); } + pipeline_cache.InvalidateRegions(sequences); } bool RasterizerVulkan::OnCPUWrite(DAddr addr, u64 size) { @@ -1424,10 +1422,27 @@ void RasterizerVulkan::FlushWork() { #endif // __ANDROID__ static_assert(DRAWS_TO_DISPATCH % (CHECK_MASK + 1) == 0); - if ((++draw_counter & CHECK_MASK) != CHECK_MASK) { + if ((++draw_counter & CHECK_MASK) != 0) { return; } - if (draw_counter < DRAWS_TO_DISPATCH) { + // Keep the host GPU working while native commands are still being recorded. + static const u32 native_flush_draws = [] { + constexpr u32 fallback = 128; + const char* value = std::getenv("EDEN_NX2_FLUSH_DRAWS"); + if (value == nullptr || value[0] < '0' || value[0] > '9') { + return fallback; + } + char* end = nullptr; + const unsigned long parsed = std::strtoul(value, &end, 10); + if (*end != '\0' || parsed == 0) { + return fallback; + } + return static_cast<u32>(std::clamp(parsed, 32UL, 1'048'576UL)); + }(); + const u32 draws_to_dispatch = Tegra::g_nx2_ampere.load(std::memory_order_relaxed) + ? native_flush_draws + : DRAWS_TO_DISPATCH; + if (draw_counter < draws_to_dispatch) { scheduler.DispatchWork(); return; } diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index d0edda81c9..e3f427dead 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -2509,23 +2509,31 @@ void TextureCacheRuntime::TickFrame() { }); } +namespace { + +// The host image of a texture; unbacked images are created without memory (see MakeImage). +[[nodiscard]] vk::Image MakeHostImage(TextureCacheRuntime& runtime, const ImageInfo& info, + u32 host_level_offset, GPUVAddr gpu_addr, bool backed) { + const Device& device = runtime.device; + const bool sparse_tiles = !backed && UsesSparseTiles(device, info, gpu_addr); + return MakeImage( + device, runtime.memory_allocator, TrimmedHostInfo(device, info, host_level_offset), + IsHugeAstcArray(device, info) ? std::span<const VkFormat>{HugeAstcViewFormats} + : IsOpaqueBc1Astc(device, info) ? std::span<const VkFormat>{OpaqueAstcViewFormats} + : runtime.ViewFormats(info.format), + AstcHostFormat(device, info), + !backed && (UsesSparseLayers(device, info, gpu_addr) || sparse_tiles || + IsUnwrittenSparse(device, info, gpu_addr)), + sparse_tiles && SparseTilesModeOf(device) == SparseTilesMode::Aliased); +} + +} // Anonymous namespace + Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu_addr_, VAddr cpu_addr_) : VideoCommon::ImageBase(info_, gpu_addr_, cpu_addr_), scheduler{&runtime_.scheduler}, runtime{&runtime_}, - original_image(MakeImage(runtime_.device, runtime_.memory_allocator, - TrimmedHostInfo(runtime_.device, info, host_level_offset), - IsHugeAstcArray(runtime_.device, info) - ? std::span<const VkFormat>{HugeAstcViewFormats} - : IsOpaqueBc1Astc(runtime_.device, info) - ? std::span<const VkFormat>{OpaqueAstcViewFormats} - : runtime->ViewFormats(info.format), - AstcHostFormat(runtime_.device, info), - UsesSparseLayers(runtime_.device, info, gpu_addr_) || - UsesSparseTiles(runtime_.device, info, gpu_addr_) || - IsUnwrittenSparse(runtime_.device, info, gpu_addr_), - UsesSparseTiles(runtime_.device, info, gpu_addr_) && - SparseTilesModeOf(runtime_.device) == SparseTilesMode::Aliased)), + original_image(MakeHostImage(runtime_, info, host_level_offset, gpu_addr_, false)), aspect_mask(ImageAspectMask(info.format)) { if (original_image.IsUnbacked()) { flags |= VideoCommon::ImageFlagBits::Unbacked; @@ -2536,6 +2544,17 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu flags |= VideoCommon::ImageFlagBits::SparseLayers; } else if (UsesSparseTiles(runtime_.device, info, gpu_addr_) && InitSparseTiles()) { flags |= VideoCommon::ImageFlagBits::SparseTiles; + } else if (UsesSparseTiles(runtime_.device, info, gpu_addr_) && + SparseTilesModeOf(runtime_.device) == SparseTilesMode::Aliased) { + // The driver can't sample unbound memory (that is why the blocks were to share one), + // so an image whose tiles could not be set up is backed whole like any other. + LOG_WARNING(Render_Vulkan, "Sparse tiled image {:#x}: backing it whole", gpu_addr_); + original_image = MakeHostImage(runtime_, info, host_level_offset, gpu_addr_, true); + if (!original_image.IsUnbacked()) { + flags &= ~VideoCommon::ImageFlagBits::Unbacked; + } else if (texels <= (u64{1} << 28)) { + flags |= VideoCommon::ImageFlagBits::UnwrittenSparse; + } } else if (texels <= (u64{1} << 28)) { // Not too big to back (see MemoryAllocator::CreateImage): unbacked because nothing // was written there yet, so the first write gets it replaced. diff --git a/src/video_core/shader_cache.cpp b/src/video_core/shader_cache.cpp index 5faed0dba9..5d41dc9697 100644 --- a/src/video_core/shader_cache.cpp +++ b/src/video_core/shader_cache.cpp @@ -6,6 +6,7 @@ #include <algorithm> #include <cstring> +#include <functional> #include <string> #include <unordered_set> #include <fmt/ranges.h> @@ -36,6 +37,17 @@ void ShaderCache::InvalidateRegion(VAddr addr, size_t size) { RemovePendingShaders(); } +void ShaderCache::InvalidateRegions(std::span<const std::pair<DAddr, size_t>> sequences) { + if (sequences.empty()) { + return; + } + std::scoped_lock lock{invalidation_mutex}; + for (const auto& [addr, size] : sequences) { + InvalidatePagesInRegion(addr, size); + } + RemovePendingShaders(); +} + void ShaderCache::OnCacheInvalidation(VAddr addr, size_t size) { std::scoped_lock lock{invalidation_mutex}; InvalidatePagesInRegion(addr, size); @@ -304,10 +316,17 @@ void ShaderCache::RemovePendingShaders() { marked_for_removal.clear(); if (!removed_shaders.empty()) { - // Remove the given shaders from the cache - std::erase_if(storage, [&removed_shaders](const std::unique_ptr<ShaderInfo>& shader) { - return std::ranges::find(removed_shaders, shader.get()) != removed_shaders.end(); - }); + if (removed_shaders.size() > 16) { + std::ranges::sort(removed_shaders, std::less<ShaderInfo*>{}); + std::erase_if(storage, [&removed_shaders](const std::unique_ptr<ShaderInfo>& shader) { + return std::binary_search(removed_shaders.begin(), removed_shaders.end(), + shader.get(), std::less<ShaderInfo*>{}); + }); + } else { + std::erase_if(storage, [&removed_shaders](const std::unique_ptr<ShaderInfo>& shader) { + return std::ranges::find(removed_shaders, shader.get()) != removed_shaders.end(); + }); + } } } diff --git a/src/video_core/shader_cache.h b/src/video_core/shader_cache.h index 281a90cb6f..7c370e217c 100644 --- a/src/video_core/shader_cache.h +++ b/src/video_core/shader_cache.h @@ -67,6 +67,8 @@ public: /// @param size Number of bytes of the invalidation void InvalidateRegion(VAddr addr, size_t size); + void InvalidateRegions(std::span<const std::pair<DAddr, size_t>> sequences); + /// @brief Unmarks a memory region as cached and marks it for removal /// @param addr Start address of the CPU write operation /// @param size Number of bytes of the CPU write operation @@ -104,6 +106,8 @@ protected: bool last_shaders_valid = false; private: + friend struct ShaderCacheTestAccess; + /// @brief Tries to obtain a cached shader starting in a given address /// @note Doesn't check for ranges, the given address has to be the start of the shader /// @param addr Start address of the shader, this doesn't cache for region diff --git a/src/video_core/shader_environment.cpp b/src/video_core/shader_environment.cpp index 22e661c849..0c7d8a0d42 100644 --- a/src/video_core/shader_environment.cpp +++ b/src/video_core/shader_environment.cpp @@ -316,7 +316,8 @@ std::optional<u64> GenericEnvironment::TryFindSize() { Tegra::Texture::TICEntry GenericEnvironment::ReadTextureInfo(GPUVAddr tic_addr, u32 tic_limit, bool via_header_index, u32 raw) { const auto handle{Tegra::Texture::TexturePair(raw, via_header_index)}; - const u32 index{Tegra::Texture::TextureHeaderIndex(handle.first, tic_limit)}; + const u32 index{Tegra::Texture::TextureHeaderIndex(handle.first, tic_limit, + gpu_memory->UsesNativeTiling())}; ASSERT(index <= tic_limit); const GPUVAddr descriptor_addr{tic_addr + index * sizeof(Tegra::Texture::TICEntry)}; Tegra::Texture::TICEntry entry; @@ -327,9 +328,9 @@ Tegra::Texture::TICEntry GenericEnvironment::ReadTextureInfo(GPUVAddr tic_addr, namespace { // Reads a native program a 4 KiB page at a time instead of one 16-byte instruction at a time. -// The reads do not flush the GPU caches first (ReadBlockUnsafe): the previous per-instruction -// ReadBlock did, once per instruction. An instruction that is not inside a fully mapped page is -// left to the caller. +// Each page is read with ReadBlock, so GPU-written code is flushed first, once per page instead +// of once per instruction. An instruction that is not inside a fully mapped page is left to the +// caller. class NativeInstructionReader { public: using Instruction = Shader::Ampere::Instruction; @@ -364,7 +365,7 @@ private: page_begin = address; page_end = address; if (length >= sizeof(Instruction) && gpu_memory.IsFullyMappedRange(address, length)) { - gpu_memory.ReadBlockUnsafe(address, page.data(), length); + gpu_memory.ReadBlock(address, page.data(), length); page_end = next_page; } } @@ -414,7 +415,7 @@ std::shared_ptr<const LoweredNativeCode> GenericEnvironment::LowerNativeCode(u32 if (!native.empty()) break; throw Shader::RuntimeError("Unmapped native shader instruction at {:#x}", address); } - gpu_memory->ReadBlockUnsafe(address, &instruction, sizeof(instruction)); + gpu_memory->ReadBlock(address, &instruction, sizeof(instruction)); } native.push_back(instruction); const auto status = capture.Observe(instruction, index); diff --git a/src/video_core/texture_cache/descriptor_table.h b/src/video_core/texture_cache/descriptor_table.h index 75383809c5..a441e572de 100644 --- a/src/video_core/texture_cache/descriptor_table.h +++ b/src/video_core/texture_cache/descriptor_table.h @@ -35,10 +35,11 @@ public: invalidated = true; } - [[nodiscard]] std::pair<T, bool> Read(Tegra::MemoryManager const& gpu_memory, u32 index) noexcept { + template <class MemoryManager> + [[nodiscard]] std::pair<T, bool> Read(const MemoryManager& gpu_memory, u32 index) noexcept { DEBUG_ASSERT(index <= current_limit); - if (invalidated) [[unlikely]] { - std::ranges::fill(read_descriptors, 0); + if (invalidated) { + ClearReadDescriptors(); invalidated = false; } const GPUVAddr gpu_addr = current_gpu_addr + index * sizeof(T); @@ -62,10 +63,8 @@ public: // Mario Brothership reallocates a lot of times, so use aggressive pre-alloc sizes // std::vector<T> by default uses quadratic growth, but that isn't even enough to satisfy brothership const size_t num_descriptors = ((limit + 0x80000) & (~0x7ffff)) + 1; - size_t old_size = read_descriptors.size(); read_descriptors.resize(Common::DivCeil(num_descriptors, 64U)); - old_size = (std::min)(old_size, read_descriptors.size()); - std::fill(read_descriptors.begin(), read_descriptors.begin() + old_size, 0); + ClearReadDescriptors(); invalidated = false; // descriptors.resize(num_descriptors); @@ -76,6 +75,13 @@ public: GPUVAddr current_gpu_addr{}; u32 current_limit{}; bool invalidated{}; ///< read_descriptors is to be cleared before the next read + +private: + void ClearReadDescriptors() noexcept { + // Reserved capacity is not addressable. A later limit increase clears its new range. + const size_t words = (std::min)(read_descriptors.size(), size_t{current_limit} / 64 + 1); + std::fill_n(read_descriptors.begin(), words, u64{0}); + } }; } // namespace VideoCommon diff --git a/src/video_core/texture_cache/format_lookup_table.cpp b/src/video_core/texture_cache/format_lookup_table.cpp index 20bb96aae3..18be2dcbf3 100644 --- a/src/video_core/texture_cache/format_lookup_table.cpp +++ b/src/video_core/texture_cache/format_lookup_table.cpp @@ -150,6 +150,7 @@ PixelFormat PixelFormatFromTextureInfo(TextureFormat format, ComponentType red, case Hash(TextureFormat::X8Z24, UNORM): return PixelFormat::X8_D24_UNORM; case Hash(TextureFormat::X8Z24, UNORM, UINT, UINT, UINT, LINEAR): + case Hash(TextureFormat::X8Z24, UNORM, UINT, UNORM, UNORM, LINEAR): return PixelFormat::X8_D24_UNORM; case Hash(TextureFormat::Z24S8, UINT, UNORM, UNORM, UNORM, LINEAR): return PixelFormat::S8_UINT_D24_UNORM; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index ecd8429fa2..abb22b9bfb 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -809,7 +809,8 @@ typename P::Framebuffer* TextureCache<P>::GetFramebuffer() { template <class P> ImageViewId TextureCache<P>::VisitImageView(u32 index, bool compute) { auto& table = compute ? channel_state->compute_image_table : channel_state->graphics_image_table; - index = Tegra::Texture::TextureHeaderIndex(index, table.current_limit); + index = Tegra::Texture::TextureHeaderIndex(index, table.current_limit, + gpu_memory->UsesNativeTiling()); if (index > table.current_limit) { LOG_DEBUG(HW_GPU, "Invalid image view index={}", index); return NULL_IMAGE_VIEW_ID; diff --git a/src/video_core/textures/texture.h b/src/video_core/textures/texture.h index 79396d8a68..35a4c8eecf 100644 --- a/src/video_core/textures/texture.h +++ b/src/video_core/textures/texture.h @@ -167,10 +167,10 @@ static_assert(sizeof(TextureHandle) == 4, "TextureHandle has wrong size"); /// Some Switch 2 bindless handles (Switch Sports' scenery, read from a uniform register) hold /// the texture header's byte offset in the pool, index * 32, where the handle normally has the -/// index. Read it as an offset only when the plain index falls past the pool and the offset -/// lands inside it, so handles that already resolve are left alone. -[[nodiscard]] constexpr u32 TextureHeaderIndex(u32 index, u32 limit) { - if (index > limit && index % 32 == 0 && index / 32 <= limit) { +/// index. Read it as an offset only on Switch 2 (native tiling), when the plain index falls past +/// the pool and the offset lands inside it, so handles that already resolve are left alone. +[[nodiscard]] constexpr u32 TextureHeaderIndex(u32 index, u32 limit, bool switch2) { + if (switch2 && index > limit && index % 32 == 0 && index / 32 <= limit) { return index / 32; } return index; diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index 7175f5dd91..edc091318c 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -1166,6 +1166,12 @@ FN_MAX_LIMIT_LIST return properties.driver.driverID == VK_DRIVER_ID_MOLTENVK; } + /// The multiple vertex input binding strides must be. Metal needs 4 bytes, which MoltenVK + /// reports as minVertexInputBindingStrideAlignment through VK_KHR_portability_subset. + u32 GetMinVertexInputBindingStrideAlignment() const noexcept { + return IsMoltenVK() ? 4 : 1; + } + NvidiaArchitecture GetNvidiaArch() const noexcept { return nvidia_arch; } |
