1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#include <boost/container/small_vector.hpp>
#include "shader_recompiler/frontend/ir/ir_emitter.h"
#include "shader_recompiler/host_translate_info.h"
#include "shader_recompiler/ir_opt/passes.h"
namespace Shader::Optimization {
namespace {
constexpr int MAX_BALLOT_DEPTH = 8;
struct SlotAtomic {
IR::Block* block;
IR::Inst* atomic;
IR::Inst* ballot;
};
IR::Inst* FindBallot(const IR::Value& value, int depth) {
if (depth > MAX_BALLOT_DEPTH || value.IsImmediate()) {
return nullptr;
}
IR::Inst* const inst{value.InstRecursive()};
if (inst->GetOpcode() == IR::Opcode::SubgroupBallot) {
return inst;
}
if (inst->GetOpcode() == IR::Opcode::Phi) {
return nullptr;
}
for (size_t index = 0; index < inst->NumArgs(); ++index) {
if (IR::Inst* const ballot{FindBallot(inst->Arg(index), depth + 1)}) {
return ballot;
}
}
return nullptr;
}
bool IsShuffle(IR::Opcode opcode) {
switch (opcode) {
case IR::Opcode::ShuffleIndex:
case IR::Opcode::ShuffleUp:
case IR::Opcode::ShuffleDown:
case IR::Opcode::ShuffleButterfly:
return true;
default:
return false;
}
}
IR::U32 PrimitiveSlot(IR::IREmitter& ir, u32 invocations, const IR::U32& amount) {
IR::U32 key{ir.GetAttributeU32(IR::Attribute::PrimitiveId)};
if (invocations > 1) {
key = ir.IAdd(ir.IMul(key, ir.Imm32(invocations)), ir.InvocationId());
}
return ir.IMul(key, amount);
}
bool MergesAtomic(const IR::Inst& phi, const IR::Block& block, const IR::Inst& atomic) {
for (size_t index = 0; index < phi.NumArgs(); ++index) {
const IR::Value arg{phi.Arg(index)};
if (phi.PhiBlock(index) == &block && !arg.IsImmediate() &&
arg.InstRecursive() == &atomic) {
return true;
}
}
return false;
}
void RewriteMergedSlots(IR::Block& block, IR::Inst& atomic, u32 invocations) {
for (IR::Block* const successor : block.ImmSuccessors()) {
for (IR::Inst& phi : successor->Instructions()) {
if (phi.GetOpcode() != IR::Opcode::Phi || !MergesAtomic(phi, block, atomic)) {
continue;
}
for (size_t index = 0; index < phi.NumArgs(); ++index) {
const IR::Value arg{phi.Arg(index)};
if (arg.IsImmediate() || !IsShuffle(arg.InstRecursive()->GetOpcode())) {
continue;
}
IR::IREmitter ir{*phi.PhiBlock(index)};
const IR::U32 amount{arg.InstRecursive()->Arg(0)};
phi.SetArg(index, PrimitiveSlot(ir, invocations, amount));
}
}
}
}
void RewriteAtomic(IR::Block& block, IR::Inst& atomic, u32 invocations) {
const auto insert_point{IR::Block::InstructionList::s_iterator_to(atomic)};
IR::IREmitter ir{block, insert_point};
const IR::U32 amount{atomic.Arg(2)};
const IR::U32 slot{PrimitiveSlot(ir, invocations, amount)};
block.PrependNewInst(insert_point, IR::Opcode::StorageAtomicUMax32,
{atomic.Arg(0), atomic.Arg(1), ir.IAdd(slot, amount)});
atomic.ReplaceUsesWith(slot);
}
void NeutralizePredicate(IR::Inst& ballot) {
const IR::Value pred{ballot.Arg(0)};
if (!pred.IsImmediate()) {
pred.InstRecursive()->ReplaceUsesWith(IR::Value{true});
}
}
}
void GeometryCompactionPass(IR::Program& program, const HostTranslateInfo& host_info) {
if (program.stage != Stage::Geometry || !host_info.single_lane_geometry_subgroups) {
return;
}
boost::container::small_vector<SlotAtomic, 4> slot_atomics;
for (IR::Block* const block : program.post_order_blocks) {
for (IR::Inst& inst : block->Instructions()) {
if (inst.GetOpcode() != IR::Opcode::StorageAtomicIAdd32) {
continue;
}
if (IR::Inst* const ballot{FindBallot(inst.Arg(2), 0)}) {
slot_atomics.push_back({block, &inst, ballot});
}
}
}
for (const SlotAtomic& slot_atomic : slot_atomics) {
RewriteMergedSlots(*slot_atomic.block, *slot_atomic.atomic, program.invocations);
RewriteAtomic(*slot_atomic.block, *slot_atomic.atomic, program.invocations);
NeutralizePredicate(*slot_atomic.ballot);
}
}
}
|
