[video_core] fix std::bitset<> dirty tracker OOB, fix slightly wrong index format estimate (#4006)

u8 may have been 0xff, (aka. 255), but bitset was only 255 elements, so doing bitset[255] is technically OOB

additionally the max size estimate for index formats was not correct, there can be up to 256 elements with a u8 format index, not just 255

Signed-off-by: lizzie <lizzie@eden-emu.dev>

Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/4006
Reviewed-by: crueter <crueter@eden-emu.dev>
Reviewed-by: MaranBr <maranbr@eden-emu.dev>

2 files changed, 14 insertions, 27 deletions

diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp
index 9aaa99f7ff..7cf351e458 100644
--- a/src/video_core/engines/maxwell_3d.cpp
+++ b/src/video_core/engines/maxwell_3d.cpp
@@ -270,31 +270,20 @@ u32 Maxwell3D::GetMaxCurrentVertices() {
 size_t Maxwell3D::EstimateIndexBufferSize() {
     GPUVAddr start_address = regs.index_buffer.StartAddress();
     GPUVAddr end_address = regs.index_buffer.EndAddress();
-    static constexpr std::array<size_t, 3> max_sizes = {(std::numeric_limits<u8>::max)(),
-                                                        (std::numeric_limits<u16>::max)(),
-                                                        (std::numeric_limits<u32>::max)()};
-    const size_t byte_size = regs.index_buffer.FormatSizeInBytes();
-    const size_t log2_byte_size = Common::Log2Ceil64(byte_size);
-    const size_t cap{GetMaxCurrentVertices() * 4 * byte_size};
-    const size_t lower_cap =
-        std::min<size_t>(static_cast<size_t>(end_address - start_address), cap);
-    return std::min<size_t>(
-        memory_manager.GetMemoryLayoutSize(start_address, byte_size * max_sizes[log2_byte_size]) /
-            byte_size,
-        lower_cap);
+    auto const byte_size = regs.index_buffer.FormatSizeInBytes();
+    auto const max_size = 1ull << (byte_size * CHAR_BIT);
+    auto const upper_cap = GetMaxCurrentVertices() * 4 * byte_size;
+    auto const lower_cap = std::min<size_t>(size_t(end_address - start_address), upper_cap);
+    return std::min<size_t>(memory_manager.GetMemoryLayoutSize(start_address, byte_size * max_size) / byte_size, lower_cap);
 }
 
 u32 Maxwell3D::ProcessShadowRam(u32 method, u32 argument) {
     // Keep track of the register value in shadow_state when requested.
-    const auto control = shadow_state.shadow_ram_control;
-    if (control == Regs::ShadowRamControl::Track ||
-        control == Regs::ShadowRamControl::TrackWithFilter) {
-        shadow_state.reg_array[method] = argument;
-        return argument;
-    }
-    if (control == Regs::ShadowRamControl::Replay) {
+    auto const c = shadow_state.shadow_ram_control;
+    if (c == Regs::ShadowRamControl::Track || c == Regs::ShadowRamControl::TrackWithFilter)
+        return shadow_state.reg_array[method] = argument;
+    else if (c == Regs::ShadowRamControl::Replay)
         return shadow_state.reg_array[method];
-    }
     return argument;
 }
 
@@ -317,10 +306,8 @@ void Maxwell3D::ConsumeSinkImpl() {
 
 void Maxwell3D::ProcessDirtyRegisters(u32 method, u32 argument) {
     regs.reg_array[method] = argument;
-
-    for (const auto& table : dirty.tables) {
+    for (auto const& table : dirty.tables)
         dirty.flags[table[method]] = true;
-    }
 }
 
 void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argument, bool is_last_call) {
diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h
index 3ac79e0eb8..864ee27fb6 100644
--- a/src/video_core/engines/maxwell_3d.h
+++ b/src/video_core/engines/maxwell_3d.h
@@ -2215,7 +2215,7 @@ public:
             u32 first;
             u32 count;
 
-            unsigned FormatSizeInBytes() const {
+            size_t FormatSizeInBytes() const {
                 switch (format) {
                 case IndexFormat::UnsignedByte:
                     return 1;
@@ -2224,7 +2224,7 @@ public:
                 case IndexFormat::UnsignedInt:
                     return 4;
                 }
-                ASSERT(false);
+                UNREACHABLE();
                 return 1;
             }
 
@@ -3148,9 +3148,9 @@ public:
     }
 
     struct DirtyState {
-        using Flags = std::bitset<(std::numeric_limits<u8>::max)()>;
+        using Flags = std::bitset<(std::numeric_limits<u8>::max)() + 1>;
         using Table = std::array<u8, Regs::NUM_REGS>;
-        using Tables = std::array<Table, 2>;
+        using Tables = std::array<std::array<u8, Regs::NUM_REGS>, 2>;
 
         Flags flags;
         Tables tables{};