Native MME: decode each instruction once; reuse the parameter FIFO

The Turing macro interpreter re-extracted every field of each 96-bit
instruction on every execution and copied the register file per
instruction; title 1's GPU thread spent about half its busy time there.
Instructions are now decoded once per PC (re-decoded when their words
change) and both lanes' operands are read up front instead of copying
the registers. The parameter FIFO is a member instead of a vector
allocated per macro call. Rendering checked unchanged on the attract
loop; the GPU thread went from ~68% to ~45-50% busy.

3 files changed, 99 insertions, 61 deletions

diff --git a/src/video_core/macro.cpp b/src/video_core/macro.cpp
index d70cfefd2f..d3ba353ca0 100644
--- a/src/video_core/macro.cpp
+++ b/src/video_core/macro.cpp
@@ -1354,7 +1354,9 @@ void MacroEngine::Execute(Core::System& system, Engines::Maxwell3D& maxwell3d, u
         }
         LOG_DEBUG(HW_GPU, "Native MME context={:#x}", context);
         LOG_DEBUG(HW_GPU, "Native MME start={:#x}, parameters={}, data={:08x}", method, parameters.size(), fmt::join(parameters, ","));
-        std::vector<u32> fifo(parameters.begin(), parameters.end());
+        // Reused between calls (a macro never calls another: methods >= 0xe00 are refused below).
+        auto& fifo = native_fifo;
+        fifo.assign(parameters.begin(), parameters.end());
         native_interpreter.active_dram = &maxwell3d.GetNativeDataRam();
         const auto result = native_interpreter.Execute(native_code, method, fifo,
             [&](u32 address) { return address < Maxwell3D::Regs::NUM_REGS ? maxwell3d.GetRegisterValue(address) : 0U; },
diff --git a/src/video_core/macro.h b/src/video_core/macro.h
index 52dfe7c82c..f744c12942 100644
--- a/src/video_core/macro.h
+++ b/src/video_core/macro.h
@@ -251,6 +251,7 @@ struct MacroEngine {
     u32 native_write_offset{};
     std::array<u32, 0x3000> native_code{};
     TuringMacroInterpreter native_interpreter;
+    std::vector<u32> native_fifo;
 
 };
 
diff --git a/src/video_core/macro_turing.h b/src/video_core/macro_turing.h
index 063e22c297..b7032eb4a4 100644
--- a/src/video_core/macro_turing.h
+++ b/src/video_core/macro_turing.h
@@ -25,6 +25,56 @@ public:
         std::string_view error;
     };
 
+    // One instruction's fields, decoded once and reused while its words stay the same (macro
+    // code runs thousands of times a frame; decoding it every time was most of the interpreter).
+    struct Decoded {
+        std::array<Word, 3> raw{~0U, ~0U, ~0U};
+        std::array<Word, 2> imm, op, dst, src_a, src_b, out_method, out_emit;
+        Word pred_reg, pred_mode;
+        // Lane/output enable masks when the predicate register is non-zero / zero.
+        Word mask_true, mask_false;
+        bool end;
+        std::array<bool, 2> load;
+    };
+
+    static Decoded Decode(const std::array<Word, 3>& bits) {
+        const auto field = [&](Word low, Word count) {
+            std::uint64_t value = bits[low / 32];
+            if (low / 32 < 2) value |= std::uint64_t{bits[low / 32 + 1]} << 32;
+            return Word((value >> (low % 32)) & ((std::uint64_t{1} << count) - 1));
+        };
+        Decoded d;
+        d.raw = bits;
+        d.imm = {field(30, 16), field(66, 16)};
+        d.op = {field(10, 5), field(46, 5)};
+        d.dst = {field(15, 5), field(51, 5)};
+        d.src_a = {field(20, 5), field(56, 5)};
+        d.src_b = {field(25, 5), field(61, 5)};
+        d.out_method = {field(82, 3), field(89, 3)};
+        d.out_emit = {field(85, 4), field(92, 4)};
+        d.pred_reg = field(5, 5);
+        d.pred_mode = field(1, 4);
+        d.end = field(0, 1) != 0;
+        constexpr std::array<std::string_view, 16> predicates{
+            "UUUU", "TTTT", "FFFF", "TTUU", "FFUU", "TFUU", "TUUU", "FUUU",
+            "UUTT", "UUTF", "UUTU", "UUFT", "UUFF", "UUFU", "UUUT", "UUUF"};
+        d.mask_true = d.mask_false = 15;
+        if (d.pred_mode) {
+            d.mask_true = d.mask_false = 0;
+            for (Word i = 0; i < 4; ++i) {
+                if (predicates[d.pred_mode][i] != 'T') d.mask_true |= 1U << i;
+                if (predicates[d.pred_mode][i] != 'F') d.mask_false |= 1U << i;
+            }
+        }
+        for (Word i = 0; i < 2; ++i) {
+            const Word reg = 28 + i, out = 3 + i;
+            d.load[i] = d.pred_reg == reg || d.src_a[0] == reg || d.src_a[1] == reg ||
+                        d.src_b[0] == reg || d.src_b[1] == reg || d.out_method[0] == out ||
+                        d.out_method[1] == out || d.out_emit[0] == out || d.out_emit[1] == out;
+        }
+        return d;
+    }
+
     template <typename Read, typename Emit>
     // `parameters` is the load FIFO; `emit` may append to it (MME_DMA_READ_FIFOED).
     Result Execute(std::span<const Word> code, Word start,
@@ -36,70 +86,53 @@ public:
         bool end_next{}, stopped{};
         std::string_view error;
         const auto fail = [&](std::string_view message) { if (error.empty()) error = message; };
+        if (decoded.size() < code.size() / 3) decoded.resize(code.size() / 3);
         for (Word steps = 0; steps < 1000000; ++steps) {
             if (static_cast<std::size_t>(pc) * 3 + 2 >= code.size()) {
                 fail("instruction outside uploaded RAM");
                 break;
             }
             const std::array<Word, 3> bits{code[pc * 3 + 2], code[pc * 3 + 1], code[pc * 3]};
-            const auto field = [&](Word low, Word count) {
-                std::uint64_t value = bits[low / 32];
-                if (low / 32 < 2) value |= std::uint64_t{bits[low / 32 + 1]} << 32;
-                return Word((value >> (low % 32)) & ((std::uint64_t{1} << count) - 1));
-            };
-            const std::array<Word, 2> imm{field(30, 16), field(66, 16)};
-            const std::array<Word, 2> op{field(10, 5), field(46, 5)};
-            const std::array<Word, 2> dst{field(15, 5), field(51, 5)};
-            const std::array<Word, 2> src_a{field(20, 5), field(56, 5)};
-            const std::array<Word, 2> src_b{field(25, 5), field(61, 5)};
-            const std::array<Word, 2> out_method{field(82, 3), field(89, 3)};
-            const std::array<Word, 2> out_emit{field(85, 4), field(92, 4)};
-            const Word pred_reg = field(5, 5), pred_mode = field(1, 4);
+            Decoded& d = decoded[pc];
+            if (d.raw != bits) d = Decode(bits);
             std::array<Word, 2> load{}, alu{};
             for (Word i = 0; i < 2; ++i) {
-                const Word reg = 28 + i, out = 3 + i;
-                const bool needed = pred_reg == reg || src_a[0] == reg || src_a[1] == reg ||
-                    src_b[0] == reg || src_b[1] == reg || out_method[0] == out ||
-                    out_method[1] == out || out_emit[0] == out || out_emit[1] == out;
-                if (needed) {
+                if (d.load[i]) {
                     if (consumed >= parameters.size()) { fail("parameter FIFO underflow"); break; }
                     load[i] = parameters[consumed++];
                 }
             }
-            // Both ALUs read the register file before either destination is
-            // committed. Carry chaining remains explicit between the lanes.
-            const auto input_regs = regs;
-            const Word input_initialized = initialized;
+            // Both ALUs read the register file before either destination is committed: their
+            // operands are read up front. Carry chaining remains explicit between the lanes.
             const auto get = [&](Word reg, Word lane) -> Word {
                 if (reg < 24) {
-                    if (!(input_initialized & (1U << reg))) fail("uninitialized register read");
-                    return input_regs[reg];
+                    if (!(initialized & (1U << reg))) fail("uninitialized register read");
+                    return regs[reg];
                 }
                 switch (reg) {
                 case 24: return 0;
-                case 25: return Word(std::int32_t(std::int16_t(imm[lane])));
-                case 26: return Word(std::int32_t(std::int16_t(imm[1 - lane])));
-                case 27: return (imm[0] << 16) | imm[1];
+                case 25: return Word(std::int32_t(std::int16_t(d.imm[lane])));
+                case 26: return Word(std::int32_t(std::int16_t(d.imm[1 - lane])));
+                case 27: return (d.imm[0] << 16) | d.imm[1];
                 case 28: return load[0];
                 case 29: return load[1];
                 default: fail("invalid source register"); return 0;
                 }
             };
-            constexpr std::array<std::string_view, 16> predicates{
-                "UUUU", "TTTT", "FFFF", "TTUU", "FFUU", "TFUU", "TUUU", "FUUU",
-                "UUTT", "UUTF", "UUTU", "UUFT", "UUFF", "UUFU", "UUUT", "UUUF"};
-            Word mask = 15;
-            if (pred_mode) {
-                const char skipped = get(pred_reg, 0) ? 'T' : 'F';
-                mask = 0;
-                for (Word i = 0; i < 4; ++i)
-                    if (predicates[pred_mode][i] != skipped) mask |= 1U << i;
+            const Word mask = d.pred_mode ? (get(d.pred_reg, 0) ? d.mask_true : d.mask_false) : 15;
+            std::array<Word, 2> xs{}, ys{};
+            for (Word lane = 0; lane < 2; ++lane) {
+                if (mask & (1U << lane)) {
+                    xs[lane] = get(d.src_a[lane], lane);
+                    ys[lane] = get(d.src_b[lane], lane);
+                }
             }
             Word next_pc = pc + 1;
             bool branch_cancels_end{};
             for (Word lane = 0; lane < 2 && error.empty(); ++lane) {
                 if (!(mask & (1U << lane))) continue;
-                const Word x = get(src_a[lane], lane), y = get(src_b[lane], lane);
+                const Word x = xs[lane], y = ys[lane];
+                const Word imm = d.imm[lane];
                 Word value{};
                 const auto condition = [&](Word comparison) {
                     switch (comparison) {
@@ -110,14 +143,14 @@ public:
                     default: return x == y;
                     }
                 };
-                switch (op[lane]) {
+                switch (d.op[lane]) {
                 case 0: value = x + y; carry = value < x; break;
                 case 1: value = x + y + carry; break;
                 case 2: value = x - y; carry = value > x; break;
                 case 3: value = x - y - carry; break;
                 case 4: case 6: {
-                    const std::uint64_t a = op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(x))) : x;
-                    const std::uint64_t b = op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(y))) : y;
+                    const std::uint64_t a = d.op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(x))) : x;
+                    const std::uint64_t b = d.op[lane] == 4 ? std::uint64_t(std::int64_t(std::int32_t(y))) : y;
                     const auto product = a * b;
                     value = Word(product); carry = Word(product >> 32); break;
                 }
@@ -136,33 +169,33 @@ public:
                 case 14: value = x | y; break;
                 case 15: value = x ^ y; break;
                 case 16: {
-                    const Word dest = (imm[lane] >> 10) & 63, count = (imm[lane] >> 5) & 31;
-                    const Word source = imm[lane] & 31;
+                    const Word dest = (imm >> 10) & 63, count = (imm >> 5) & 31;
+                    const Word source = imm & 31;
                     if (dest >= 32) { fail("invalid merge destination"); break; }
                     const Word bits_mask = (1U << count) - 1;
                     value = (x & ~(bits_mask << dest)) | (((y >> source) & bits_mask) << dest);
                     break;
                 }
                 case 17: case 18: case 19: case 20: case 21:
-                    value = condition(op[lane]) ? ~0U : 0; break;
+                    value = condition(d.op[lane]) ? ~0U : 0; break;
                 case 22: value = read((x + y) & 0xffff); break;
                 case 23:
-                    if (loop_count || imm[lane] < 2) { fail("invalid loop"); break; }
+                    if (loop_count || imm < 2) { fail("invalid loop"); break; }
                     // The native SDK emits zero-count loops around FIFO reads.
                     // Unlike Mesa's test simulator, these must skip the body.
-                    if (x == 0) { next_pc = pc + imm[lane]; break; }
-                    loop_count = x - 1; loop_start = pc; loop_end = pc + imm[lane] - 1;
+                    if (x == 0) { next_pc = pc + imm; break; }
+                    loop_count = x - 1; loop_start = pc; loop_end = pc + imm - 1;
                     break;
-                case 24: next_pc = pc + (imm[lane] & 0x7fff); break;
+                case 24: next_pc = pc + (imm & 0x7fff); break;
                 case 25: case 26: case 27: case 28: case 29:
-                    if (condition(op[lane]) == bool(imm[lane] & 0x8000)) {
-                        const auto offset = std::int32_t(imm[lane] << 19) >> 19;
+                    if (condition(d.op[lane]) == bool(imm & 0x8000)) {
+                        const auto offset = std::int32_t(imm << 19) >> 19;
                         if (offset == -4096) stopped = true;
                         else {
                             next_pc = Word(std::int32_t(pc) + offset);
                             // Native back edges use bit 14 to suppress the
                             // fallthrough termination when the branch is taken.
-                            branch_cancels_end = (imm[lane] & 0x4000) != 0;
+                            branch_cancels_end = (imm & 0x4000) != 0;
                         }
                     }
                     break;
@@ -174,34 +207,34 @@ public:
                     (*active_dram)[x] = y; break;
                 }
                 alu[lane] = value;
-                if (dst[lane] < 24) { regs[dst[lane]] = value; initialized |= 1U << dst[lane]; }
-                else if (dst[lane] != 24) fail("invalid destination register");
+                if (d.dst[lane] < 24) { regs[d.dst[lane]] = value; initialized |= 1U << d.dst[lane]; }
+                else if (d.dst[lane] != 24) fail("invalid destination register");
             }
             const auto output = [&](Word selector) -> Word {
                 switch (selector) {
                 case 1: return alu[0]; case 2: return alu[1];
                 case 3: return load[0]; case 4: return load[1];
-                case 5: return imm[0]; case 6: return imm[1];
-                case 8: return imm[0] >> 12; case 9: return imm[1] >> 12;
-                case 10: return (imm[0] << 16) | imm[1];
+                case 5: return d.imm[0]; case 6: return d.imm[1];
+                case 8: return d.imm[0] >> 12; case 9: return d.imm[1] >> 12;
+                case 10: return (d.imm[0] << 16) | d.imm[1];
                 default: fail("invalid output selector"); return 0;
                 }
             };
             for (Word lane = 0; lane < 2 && error.empty(); ++lane) {
                 if (!(mask & (4U << lane))) continue;
-                if (out_method[lane]) {
-                    const Word address = output(out_method[lane]);
+                if (d.out_method[lane]) {
+                    const Word address = output(d.out_method[lane]);
                     method = address & 0xfff; increment = (address >> 12) & 15;
                 }
-                if (out_emit[lane]) {
-                    const Word value = output(out_emit[lane]);
+                if (d.out_emit[lane]) {
+                    const Word value = output(d.out_emit[lane]);
                     if (error.empty() && !emit(method, value)) fail("unsupported emitted method");
                     method += increment;
                 }
             }
             if (!error.empty()) break;
             if (end_next || stopped) return {true, pc, consumed, {}};
-            end_next = field(0, 1) != 0 && !branch_cancels_end;
+            end_next = d.end && !branch_cancels_end;
             if (loop_count && pc == loop_end) { --loop_count; next_pc = loop_start + 1; }
             pc = next_pc;
         }
@@ -215,6 +248,8 @@ public:
     // shared by the channels of a process, but each channel keeps its own data RAM (title 1's
     // channels store their own QMD ring base in word 0x27).
     DataRam* active_dram{&dram};
+    // Decoded instructions by PC, re-decoded whenever their words differ.
+    std::vector<Decoded> decoded;
 };
 
 } // namespace Tegra