1 #include "sfn_emitssboinstruction.h"
3 #include "sfn_instruction_fetch.h"
4 #include "sfn_instruction_gds.h"
5 #include "sfn_instruction_misc.h"
6 #include "../r600_pipe.h"
10 bool EmitSSBOInstruction::do_emit(nir_instr
* instr
)
12 const nir_intrinsic_instr
*intr
= nir_instr_as_intrinsic(instr
);
13 switch (intr
->intrinsic
) {
14 case nir_intrinsic_atomic_counter_add
:
15 case nir_intrinsic_atomic_counter_and
:
16 case nir_intrinsic_atomic_counter_exchange
:
17 case nir_intrinsic_atomic_counter_max
:
18 case nir_intrinsic_atomic_counter_min
:
19 case nir_intrinsic_atomic_counter_or
:
20 case nir_intrinsic_atomic_counter_xor
:
21 case nir_intrinsic_atomic_counter_comp_swap
:
22 return emit_atomic(intr
);
23 case nir_intrinsic_atomic_counter_read
:
24 case nir_intrinsic_atomic_counter_post_dec
:
25 return emit_unary_atomic(intr
);
26 case nir_intrinsic_atomic_counter_inc
:
27 return emit_atomic_inc(intr
);
28 case nir_intrinsic_atomic_counter_pre_dec
:
29 return emit_atomic_pre_dec(intr
);
30 case nir_intrinsic_load_ssbo
:
31 return emit_load_ssbo(intr
);
32 case nir_intrinsic_store_ssbo
:
33 return emit_store_ssbo(intr
);
39 bool EmitSSBOInstruction::emit_atomic(const nir_intrinsic_instr
* instr
)
41 ESDOp op
= get_opcode(instr
->intrinsic
);
43 if (DS_OP_INVALID
== op
)
46 GPRVector dest
= make_dest(instr
);
48 int base
= nir_intrinsic_base(instr
);
50 PValue uav_id
= from_nir(instr
->src
[0], 0);
52 PValue value
= from_nir_with_fetch_constant(instr
->src
[1], 0);
54 GDSInstr
*ir
= nullptr;
55 if (instr
->intrinsic
== nir_intrinsic_atomic_counter_comp_swap
) {
56 PValue value2
= from_nir_with_fetch_constant(instr
->src
[1], 1);
57 ir
= new GDSInstr(op
, dest
, value
, value2
, uav_id
, base
);
59 ir
= new GDSInstr(op
, dest
, value
, uav_id
, base
);
66 bool EmitSSBOInstruction::emit_unary_atomic(const nir_intrinsic_instr
* instr
)
68 ESDOp op
= get_opcode(instr
->intrinsic
);
70 if (DS_OP_INVALID
== op
)
73 GPRVector dest
= make_dest(instr
);
75 PValue uav_id
= from_nir(instr
->src
[0], 0);
77 auto ir
= new GDSInstr(op
, dest
, uav_id
, nir_intrinsic_base(instr
));
83 ESDOp
EmitSSBOInstruction::get_opcode(const nir_intrinsic_op opcode
)
86 case nir_intrinsic_atomic_counter_add
:
88 case nir_intrinsic_atomic_counter_and
:
90 case nir_intrinsic_atomic_counter_exchange
:
91 return DS_OP_XCHG_RET
;
92 case nir_intrinsic_atomic_counter_inc
:
94 case nir_intrinsic_atomic_counter_max
:
95 return DS_OP_MAX_UINT_RET
;
96 case nir_intrinsic_atomic_counter_min
:
97 return DS_OP_MIN_UINT_RET
;
98 case nir_intrinsic_atomic_counter_or
:
100 case nir_intrinsic_atomic_counter_read
:
101 return DS_OP_READ_RET
;
102 case nir_intrinsic_atomic_counter_xor
:
103 return DS_OP_XOR_RET
;
104 case nir_intrinsic_atomic_counter_post_dec
:
105 return DS_OP_DEC_RET
;
106 case nir_intrinsic_atomic_counter_comp_swap
:
107 return DS_OP_CMP_XCHG_RET
;
108 case nir_intrinsic_atomic_counter_pre_dec
:
110 return DS_OP_INVALID
;
115 bool EmitSSBOInstruction::emit_atomic_add(const nir_intrinsic_instr
* instr
)
117 GPRVector dest
= make_dest(instr
);
119 PValue value
= from_nir_with_fetch_constant(instr
->src
[1], 0);
121 PValue uav_id
= from_nir(instr
->src
[0], 0);
123 auto ir
= new GDSInstr(DS_OP_ADD_RET
, dest
, value
, uav_id
,
124 nir_intrinsic_base(instr
));
126 emit_instruction(ir
);
130 bool EmitSSBOInstruction::emit_atomic_inc(const nir_intrinsic_instr
* instr
)
132 GPRVector dest
= make_dest(instr
);
134 PValue uav_id
= from_nir(instr
->src
[0], 0);
137 if (!m_atomic_limit
) {
138 int one_tmp
= allocate_temp_register();
139 m_atomic_limit
= PValue(new GPRValue(one_tmp
, 0));
140 emit_instruction(new AluInstruction(op1_mov
, m_atomic_limit
,
141 PValue(new LiteralValue(0xffffffff)),
142 {alu_write
, alu_last_instr
}));
145 auto ir
= new GDSInstr(DS_OP_INC_RET
, dest
, m_atomic_limit
, uav_id
,
146 nir_intrinsic_base(instr
));
147 emit_instruction(ir
);
151 bool EmitSSBOInstruction::emit_atomic_pre_dec(const nir_intrinsic_instr
*instr
)
153 GPRVector dest
= make_dest(instr
);
155 PValue uav_id
= from_nir(instr
->src
[0], 0);
157 int one_tmp
= allocate_temp_register();
158 PValue
value(new GPRValue(one_tmp
, 0));
159 emit_instruction(new AluInstruction(op1_mov
, value
, Value::one_i
,
160 {alu_write
, alu_last_instr
}));
162 auto ir
= new GDSInstr(DS_OP_SUB_RET
, dest
, value
, uav_id
,
163 nir_intrinsic_base(instr
));
164 emit_instruction(ir
);
166 ir
= new GDSInstr(DS_OP_READ_RET
, dest
, uav_id
, nir_intrinsic_base(instr
));
167 emit_instruction(ir
);
172 bool EmitSSBOInstruction::emit_load_ssbo(const nir_intrinsic_instr
* instr
)
174 GPRVector dest
= make_dest(instr
);
176 /** src0 not used, should be some offset */
177 auto addr
= from_nir_with_fetch_constant(instr
->src
[1], 0);
178 PValue addr_temp
= create_register_from_nir_src(instr
->src
[1], 1);
180 /** Should be lowered in nir */
181 emit_instruction(new AluInstruction(op2_lshr_int
, addr_temp
, {addr
, PValue(new LiteralValue(2))},
182 {alu_write
, alu_last_instr
}));
184 const EVTXDataFormat formats
[4] = {
191 const std::array
<int,4> dest_swt
[4] = {
198 /* TODO fix resource index */
199 auto ir
= new FetchInstruction(dest
, addr_temp
,
200 R600_IMAGE_REAL_RESOURCE_OFFSET
, from_nir(instr
->src
[0], 0),
201 formats
[instr
->num_components
-1], vtx_nf_int
);
202 ir
->set_dest_swizzle(dest_swt
[instr
->num_components
- 1]);
203 ir
->set_flag(vtx_use_tc
);
205 emit_instruction(ir
);
209 bool EmitSSBOInstruction::emit_store_ssbo(const nir_intrinsic_instr
* instr
)
212 GPRVector::Swizzle swz
= {7,7,7,7};
213 for (int i
= 0; i
< instr
->src
[0].ssa
->num_components
; ++i
)
216 auto orig_addr
= from_nir(instr
->src
[2], 0);
218 int temp1
= allocate_temp_register();
219 GPRVector
addr_vec(temp1
, {0,1,2,7});
221 auto rat_id
= from_nir(instr
->src
[1], 0);
223 emit_instruction(new AluInstruction(op2_lshr_int
, addr_vec
.reg_i(0), orig_addr
,
224 PValue(new LiteralValue(2)), write
));
225 emit_instruction(new AluInstruction(op1_mov
, addr_vec
.reg_i(1), Value::zero
, write
));
226 emit_instruction(new AluInstruction(op1_mov
, addr_vec
.reg_i(2), Value::zero
, last_write
));
229 //#define WRITE_AS_VECTOR
230 #ifdef WRITE_AS_VECTOR
231 std::unique_ptr
<GPRVector
> value(vec_from_nir_with_fetch_constant(instr
->src
[0],
232 (1 << instr
->src
[0].ssa
->num_components
) - 1, swz
));
234 /* TODO fix resource index */
235 int nelements
= instr
->src
[0].ssa
->num_components
- 1;
238 auto ir
= new RatInstruction(cf_mem_rat
, RatInstruction::STORE_TYPED
,
239 *value
, addr_vec
, 0, rat_id
, 11,
240 (1 << instr
->src
[0].ssa
->num_components
) - 1,
242 emit_instruction(ir
);
245 PValue
value(from_nir_with_fetch_constant(instr
->src
[0], 0));
246 GPRVector
out_vec({value
, value
, value
, value
});
247 emit_instruction(new RatInstruction(cf_mem_rat
, RatInstruction::STORE_TYPED
,
248 out_vec
, addr_vec
, 0, rat_id
, 1,
250 for (int i
= 1; i
< instr
->src
[0].ssa
->num_components
; ++i
) {
251 emit_instruction(new AluInstruction(op1_mov
, out_vec
.reg_i(0), from_nir(instr
->src
[0], i
), write
));
252 emit_instruction(new AluInstruction(op2_add_int
, addr_vec
.reg_i(0),
253 {addr_vec
.reg_i(0), Value::one_i
}, last_write
));
254 emit_instruction(new RatInstruction(cf_mem_rat
, RatInstruction::STORE_TYPED
,
255 out_vec
, addr_vec
, 0, rat_id
, 1,
262 GPRVector
EmitSSBOInstruction::make_dest(const nir_intrinsic_instr
* ir
)
266 for (i
= 0; i
< 4; ++i
)
267 v
[i
] = from_nir(ir
->dest
, i
);