2 * Copyright © 2018 Valve Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Daniel Schürmann (daniel.schuermann@campus.tu-berlin.de)
31 #include "aco_builder.h"
32 #include "util/u_math.h"
34 #include "vulkan/radv_shader.h"
39 struct lower_context
{
41 std::vector
<aco_ptr
<Instruction
>> instructions
;
44 aco_opcode
get_reduce_opcode(chip_class chip
, ReduceOp op
) {
45 /* Because some 16-bit instructions are already VOP3 on GFX10, we use the
46 * 32-bit opcodes (VOP2) which allows to remove the tempory VGPR and to use
47 * DPP with the arithmetic instructions. This requires to sign-extend.
53 return aco_opcode::v_add_u32
;
54 } else if (chip
>= GFX8
) {
55 return aco_opcode::v_add_u16
;
57 return aco_opcode::v_add_co_u32
;
63 return aco_opcode::v_mul_lo_u16_e64
;
64 } else if (chip
>= GFX8
) {
65 return aco_opcode::v_mul_lo_u16
;
67 return aco_opcode::v_mul_u32_u24
;
70 case fadd16
: return aco_opcode::v_add_f16
;
71 case fmul16
: return aco_opcode::v_mul_f16
;
75 return aco_opcode::v_max_i32
;
76 } else if (chip
>= GFX8
) {
77 return aco_opcode::v_max_i16
;
79 return aco_opcode::v_max_i32
;
85 return aco_opcode::v_min_i32
;
86 } else if (chip
>= GFX8
) {
87 return aco_opcode::v_min_i16
;
89 return aco_opcode::v_min_i32
;
95 return aco_opcode::v_min_u32
;
96 } else if (chip
>= GFX8
) {
97 return aco_opcode::v_min_u16
;
99 return aco_opcode::v_min_u32
;
105 return aco_opcode::v_max_u32
;
106 } else if (chip
>= GFX8
) {
107 return aco_opcode::v_max_u16
;
109 return aco_opcode::v_max_u32
;
112 case fmin16
: return aco_opcode::v_min_f16
;
113 case fmax16
: return aco_opcode::v_max_f16
;
114 case iadd32
: return chip
>= GFX9
? aco_opcode::v_add_u32
: aco_opcode::v_add_co_u32
;
115 case imul32
: return aco_opcode::v_mul_lo_u32
;
116 case fadd32
: return aco_opcode::v_add_f32
;
117 case fmul32
: return aco_opcode::v_mul_f32
;
118 case imax32
: return aco_opcode::v_max_i32
;
119 case imin32
: return aco_opcode::v_min_i32
;
120 case umin32
: return aco_opcode::v_min_u32
;
121 case umax32
: return aco_opcode::v_max_u32
;
122 case fmin32
: return aco_opcode::v_min_f32
;
123 case fmax32
: return aco_opcode::v_max_f32
;
126 case iand32
: return aco_opcode::v_and_b32
;
129 case ixor32
: return aco_opcode::v_xor_b32
;
132 case ior32
: return aco_opcode::v_or_b32
;
133 case iadd64
: return aco_opcode::num_opcodes
;
134 case imul64
: return aco_opcode::num_opcodes
;
135 case fadd64
: return aco_opcode::v_add_f64
;
136 case fmul64
: return aco_opcode::v_mul_f64
;
137 case imin64
: return aco_opcode::num_opcodes
;
138 case imax64
: return aco_opcode::num_opcodes
;
139 case umin64
: return aco_opcode::num_opcodes
;
140 case umax64
: return aco_opcode::num_opcodes
;
141 case fmin64
: return aco_opcode::v_min_f64
;
142 case fmax64
: return aco_opcode::v_max_f64
;
143 case iand64
: return aco_opcode::num_opcodes
;
144 case ior64
: return aco_opcode::num_opcodes
;
145 case ixor64
: return aco_opcode::num_opcodes
;
146 default: return aco_opcode::num_opcodes
;
150 bool is_vop3_reduce_opcode(aco_opcode opcode
)
152 /* 64-bit reductions are VOP3. */
153 if (opcode
== aco_opcode::num_opcodes
)
156 return instr_info
.format
[(int)opcode
] == Format::VOP3
;
159 void emit_vadd32(Builder
& bld
, Definition def
, Operand src0
, Operand src1
)
161 Instruction
*instr
= bld
.vadd32(def
, src0
, src1
, false, Operand(s2
), true);
162 if (instr
->definitions
.size() >= 2) {
163 assert(instr
->definitions
[1].regClass() == bld
.lm
);
164 instr
->definitions
[1].setFixed(vcc
);
168 void emit_int64_dpp_op(lower_context
*ctx
, PhysReg dst_reg
, PhysReg src0_reg
, PhysReg src1_reg
,
169 PhysReg vtmp_reg
, ReduceOp op
,
170 unsigned dpp_ctrl
, unsigned row_mask
, unsigned bank_mask
, bool bound_ctrl
,
171 Operand
*identity
=NULL
)
173 Builder
bld(ctx
->program
, &ctx
->instructions
);
174 Definition dst
[] = {Definition(dst_reg
, v1
), Definition(PhysReg
{dst_reg
+1}, v1
)};
175 Definition vtmp_def
[] = {Definition(vtmp_reg
, v1
), Definition(PhysReg
{vtmp_reg
+1}, v1
)};
176 Operand src0
[] = {Operand(src0_reg
, v1
), Operand(PhysReg
{src0_reg
+1}, v1
)};
177 Operand src1
[] = {Operand(src1_reg
, v1
), Operand(PhysReg
{src1_reg
+1}, v1
)};
178 Operand src1_64
= Operand(src1_reg
, v2
);
179 Operand vtmp_op
[] = {Operand(vtmp_reg
, v1
), Operand(PhysReg
{vtmp_reg
+1}, v1
)};
180 Operand vtmp_op64
= Operand(vtmp_reg
, v2
);
182 if (ctx
->program
->chip_class
>= GFX10
) {
184 bld
.vop1(aco_opcode::v_mov_b32
, vtmp_def
[0], identity
[0]);
185 bld
.vop1_dpp(aco_opcode::v_mov_b32
, vtmp_def
[0], src0
[0],
186 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
187 bld
.vop3(aco_opcode::v_add_co_u32_e64
, dst
[0], bld
.def(bld
.lm
, vcc
), vtmp_op
[0], src1
[0]);
189 bld
.vop2_dpp(aco_opcode::v_add_co_u32
, dst
[0], bld
.def(bld
.lm
, vcc
), src0
[0], src1
[0],
190 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
192 bld
.vop2_dpp(aco_opcode::v_addc_co_u32
, dst
[1], bld
.def(bld
.lm
, vcc
), src0
[1], src1
[1], Operand(vcc
, bld
.lm
),
193 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
194 } else if (op
== iand64
) {
195 bld
.vop2_dpp(aco_opcode::v_and_b32
, dst
[0], src0
[0], src1
[0],
196 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
197 bld
.vop2_dpp(aco_opcode::v_and_b32
, dst
[1], src0
[1], src1
[1],
198 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
199 } else if (op
== ior64
) {
200 bld
.vop2_dpp(aco_opcode::v_or_b32
, dst
[0], src0
[0], src1
[0],
201 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
202 bld
.vop2_dpp(aco_opcode::v_or_b32
, dst
[1], src0
[1], src1
[1],
203 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
204 } else if (op
== ixor64
) {
205 bld
.vop2_dpp(aco_opcode::v_xor_b32
, dst
[0], src0
[0], src1
[0],
206 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
207 bld
.vop2_dpp(aco_opcode::v_xor_b32
, dst
[1], src0
[1], src1
[1],
208 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
209 } else if (op
== umin64
|| op
== umax64
|| op
== imin64
|| op
== imax64
) {
210 aco_opcode cmp
= aco_opcode::num_opcodes
;
213 cmp
= aco_opcode::v_cmp_gt_u64
;
216 cmp
= aco_opcode::v_cmp_lt_u64
;
219 cmp
= aco_opcode::v_cmp_gt_i64
;
222 cmp
= aco_opcode::v_cmp_lt_i64
;
229 bld
.vop1(aco_opcode::v_mov_b32
, vtmp_def
[0], identity
[0]);
230 bld
.vop1(aco_opcode::v_mov_b32
, vtmp_def
[1], identity
[1]);
232 bld
.vop1_dpp(aco_opcode::v_mov_b32
, vtmp_def
[0], src0
[0],
233 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
234 bld
.vop1_dpp(aco_opcode::v_mov_b32
, vtmp_def
[1], src0
[1],
235 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
237 bld
.vopc(cmp
, bld
.def(bld
.lm
, vcc
), vtmp_op64
, src1_64
);
238 bld
.vop2(aco_opcode::v_cndmask_b32
, dst
[0], vtmp_op
[0], src1
[0], Operand(vcc
, bld
.lm
));
239 bld
.vop2(aco_opcode::v_cndmask_b32
, dst
[1], vtmp_op
[1], src1
[1], Operand(vcc
, bld
.lm
));
240 } else if (op
== imul64
) {
242 * t1 = umul_lo(t4, y_lo)
244 * t0 = umul_lo(t3, y_hi)
246 * t5 = umul_hi(t3, y_lo)
247 * res_hi = iadd(t2, t5)
248 * res_lo = umul_lo(t3, y_lo)
249 * Requires that res_hi != src0[0] and res_hi != src1[0]
250 * and that vtmp[0] != res_hi.
253 bld
.vop1(aco_opcode::v_mov_b32
, vtmp_def
[0], identity
[1]);
254 bld
.vop1_dpp(aco_opcode::v_mov_b32
, vtmp_def
[0], src0
[1],
255 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
256 bld
.vop3(aco_opcode::v_mul_lo_u32
, vtmp_def
[1], vtmp_op
[0], src1
[0]);
258 bld
.vop1(aco_opcode::v_mov_b32
, vtmp_def
[0], identity
[0]);
259 bld
.vop1_dpp(aco_opcode::v_mov_b32
, vtmp_def
[0], src0
[0],
260 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
261 bld
.vop3(aco_opcode::v_mul_lo_u32
, vtmp_def
[0], vtmp_op
[0], src1
[1]);
262 emit_vadd32(bld
, vtmp_def
[1], vtmp_op
[0], vtmp_op
[1]);
264 bld
.vop1(aco_opcode::v_mov_b32
, vtmp_def
[0], identity
[0]);
265 bld
.vop1_dpp(aco_opcode::v_mov_b32
, vtmp_def
[0], src0
[0],
266 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
267 bld
.vop3(aco_opcode::v_mul_hi_u32
, vtmp_def
[0], vtmp_op
[0], src1
[0]);
268 emit_vadd32(bld
, dst
[1], vtmp_op
[1], vtmp_op
[0]);
270 bld
.vop1(aco_opcode::v_mov_b32
, vtmp_def
[0], identity
[0]);
271 bld
.vop1_dpp(aco_opcode::v_mov_b32
, vtmp_def
[0], src0
[0],
272 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
273 bld
.vop3(aco_opcode::v_mul_lo_u32
, dst
[0], vtmp_op
[0], src1
[0]);
277 void emit_int64_op(lower_context
*ctx
, PhysReg dst_reg
, PhysReg src0_reg
, PhysReg src1_reg
, PhysReg vtmp
, ReduceOp op
)
279 Builder
bld(ctx
->program
, &ctx
->instructions
);
280 Definition dst
[] = {Definition(dst_reg
, v1
), Definition(PhysReg
{dst_reg
+1}, v1
)};
281 RegClass src0_rc
= src0_reg
.reg() >= 256 ? v1
: s1
;
282 Operand src0
[] = {Operand(src0_reg
, src0_rc
), Operand(PhysReg
{src0_reg
+1}, src0_rc
)};
283 Operand src1
[] = {Operand(src1_reg
, v1
), Operand(PhysReg
{src1_reg
+1}, v1
)};
284 Operand src0_64
= Operand(src0_reg
, src0_reg
.reg() >= 256 ? v2
: s2
);
285 Operand src1_64
= Operand(src1_reg
, v2
);
288 (op
== imul64
|| op
== umin64
|| op
== umax64
|| op
== imin64
|| op
== imax64
)) {
289 assert(vtmp
.reg() != 0);
290 bld
.vop1(aco_opcode::v_mov_b32
, Definition(vtmp
, v1
), src0
[0]);
291 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{vtmp
+1}, v1
), src0
[1]);
293 src0
[0] = Operand(vtmp
, v1
);
294 src0
[1] = Operand(PhysReg
{vtmp
+1}, v1
);
295 src0_64
= Operand(vtmp
, v2
);
296 } else if (src0_rc
== s1
&& op
== iadd64
) {
297 assert(vtmp
.reg() != 0);
298 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{vtmp
+1}, v1
), src0
[1]);
299 src0
[1] = Operand(PhysReg
{vtmp
+1}, v1
);
303 if (ctx
->program
->chip_class
>= GFX10
) {
304 bld
.vop3(aco_opcode::v_add_co_u32_e64
, dst
[0], bld
.def(bld
.lm
, vcc
), src0
[0], src1
[0]);
306 bld
.vop2(aco_opcode::v_add_co_u32
, dst
[0], bld
.def(bld
.lm
, vcc
), src0
[0], src1
[0]);
308 bld
.vop2(aco_opcode::v_addc_co_u32
, dst
[1], bld
.def(bld
.lm
, vcc
), src0
[1], src1
[1], Operand(vcc
, bld
.lm
));
309 } else if (op
== iand64
) {
310 bld
.vop2(aco_opcode::v_and_b32
, dst
[0], src0
[0], src1
[0]);
311 bld
.vop2(aco_opcode::v_and_b32
, dst
[1], src0
[1], src1
[1]);
312 } else if (op
== ior64
) {
313 bld
.vop2(aco_opcode::v_or_b32
, dst
[0], src0
[0], src1
[0]);
314 bld
.vop2(aco_opcode::v_or_b32
, dst
[1], src0
[1], src1
[1]);
315 } else if (op
== ixor64
) {
316 bld
.vop2(aco_opcode::v_xor_b32
, dst
[0], src0
[0], src1
[0]);
317 bld
.vop2(aco_opcode::v_xor_b32
, dst
[1], src0
[1], src1
[1]);
318 } else if (op
== umin64
|| op
== umax64
|| op
== imin64
|| op
== imax64
) {
319 aco_opcode cmp
= aco_opcode::num_opcodes
;
322 cmp
= aco_opcode::v_cmp_gt_u64
;
325 cmp
= aco_opcode::v_cmp_lt_u64
;
328 cmp
= aco_opcode::v_cmp_gt_i64
;
331 cmp
= aco_opcode::v_cmp_lt_i64
;
337 bld
.vopc(cmp
, bld
.def(bld
.lm
, vcc
), src0_64
, src1_64
);
338 bld
.vop2(aco_opcode::v_cndmask_b32
, dst
[0], src0
[0], src1
[0], Operand(vcc
, bld
.lm
));
339 bld
.vop2(aco_opcode::v_cndmask_b32
, dst
[1], src0
[1], src1
[1], Operand(vcc
, bld
.lm
));
340 } else if (op
== imul64
) {
341 if (src1_reg
== dst_reg
) {
342 /* it's fine if src0==dst but not if src1==dst */
343 std::swap(src0_reg
, src1_reg
);
344 std::swap(src0
[0], src1
[0]);
345 std::swap(src0
[1], src1
[1]);
346 std::swap(src0_64
, src1_64
);
348 assert(!(src0_reg
== src1_reg
));
349 /* t1 = umul_lo(x_hi, y_lo)
350 * t0 = umul_lo(x_lo, y_hi)
352 * t5 = umul_hi(x_lo, y_lo)
353 * res_hi = iadd(t2, t5)
354 * res_lo = umul_lo(x_lo, y_lo)
355 * assumes that it's ok to modify x_hi/y_hi, since we might not have vtmp
357 Definition
tmp0_def(PhysReg
{src0_reg
+1}, v1
);
358 Definition
tmp1_def(PhysReg
{src1_reg
+1}, v1
);
359 Operand tmp0_op
= src0
[1];
360 Operand tmp1_op
= src1
[1];
361 bld
.vop3(aco_opcode::v_mul_lo_u32
, tmp0_def
, src0
[1], src1
[0]);
362 bld
.vop3(aco_opcode::v_mul_lo_u32
, tmp1_def
, src0
[0], src1
[1]);
363 emit_vadd32(bld
, tmp0_def
, tmp1_op
, tmp0_op
);
364 bld
.vop3(aco_opcode::v_mul_hi_u32
, tmp1_def
, src0
[0], src1
[0]);
365 emit_vadd32(bld
, dst
[1], tmp0_op
, tmp1_op
);
366 bld
.vop3(aco_opcode::v_mul_lo_u32
, dst
[0], src0
[0], src1
[0]);
370 void emit_dpp_op(lower_context
*ctx
, PhysReg dst_reg
, PhysReg src0_reg
, PhysReg src1_reg
,
371 PhysReg vtmp
, ReduceOp op
, unsigned size
,
372 unsigned dpp_ctrl
, unsigned row_mask
, unsigned bank_mask
, bool bound_ctrl
,
373 Operand
*identity
=NULL
) /* for VOP3 with sparse writes */
375 Builder
bld(ctx
->program
, &ctx
->instructions
);
376 RegClass rc
= RegClass(RegType::vgpr
, size
);
377 Definition
dst(dst_reg
, rc
);
378 Operand
src0(src0_reg
, rc
);
379 Operand
src1(src1_reg
, rc
);
381 aco_opcode opcode
= get_reduce_opcode(ctx
->program
->chip_class
, op
);
382 bool vop3
= is_vop3_reduce_opcode(opcode
);
385 if (opcode
== aco_opcode::v_add_co_u32
)
386 bld
.vop2_dpp(opcode
, dst
, bld
.def(bld
.lm
, vcc
), src0
, src1
, dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
388 bld
.vop2_dpp(opcode
, dst
, src0
, src1
, dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
392 if (opcode
== aco_opcode::num_opcodes
) {
393 emit_int64_dpp_op(ctx
, dst_reg
,src0_reg
, src1_reg
, vtmp
, op
,
394 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
, identity
);
399 bld
.vop1(aco_opcode::v_mov_b32
, Definition(vtmp
, v1
), identity
[0]);
400 if (identity
&& size
>= 2)
401 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{vtmp
+1}, v1
), identity
[1]);
403 for (unsigned i
= 0; i
< size
; i
++)
404 bld
.vop1_dpp(aco_opcode::v_mov_b32
, Definition(PhysReg
{vtmp
+i
}, v1
), Operand(PhysReg
{src0_reg
+i
}, v1
),
405 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
407 bld
.vop3(opcode
, dst
, Operand(vtmp
, rc
), src1
);
410 void emit_op(lower_context
*ctx
, PhysReg dst_reg
, PhysReg src0_reg
, PhysReg src1_reg
,
411 PhysReg vtmp
, ReduceOp op
, unsigned size
)
413 Builder
bld(ctx
->program
, &ctx
->instructions
);
414 RegClass rc
= RegClass(RegType::vgpr
, size
);
415 Definition
dst(dst_reg
, rc
);
416 Operand
src0(src0_reg
, RegClass(src0_reg
.reg() >= 256 ? RegType::vgpr
: RegType::sgpr
, size
));
417 Operand
src1(src1_reg
, rc
);
419 aco_opcode opcode
= get_reduce_opcode(ctx
->program
->chip_class
, op
);
420 bool vop3
= is_vop3_reduce_opcode(opcode
);
422 if (opcode
== aco_opcode::num_opcodes
) {
423 emit_int64_op(ctx
, dst_reg
, src0_reg
, src1_reg
, vtmp
, op
);
428 bld
.vop3(opcode
, dst
, src0
, src1
);
429 } else if (opcode
== aco_opcode::v_add_co_u32
) {
430 bld
.vop2(opcode
, dst
, bld
.def(bld
.lm
, vcc
), src0
, src1
);
432 bld
.vop2(opcode
, dst
, src0
, src1
);
436 void emit_dpp_mov(lower_context
*ctx
, PhysReg dst
, PhysReg src0
, unsigned size
,
437 unsigned dpp_ctrl
, unsigned row_mask
, unsigned bank_mask
, bool bound_ctrl
)
439 Builder
bld(ctx
->program
, &ctx
->instructions
);
440 for (unsigned i
= 0; i
< size
; i
++) {
441 bld
.vop1_dpp(aco_opcode::v_mov_b32
, Definition(PhysReg
{dst
+i
}, v1
), Operand(PhysReg
{src0
+i
}, v1
),
442 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
446 uint32_t get_reduction_identity(ReduceOp op
, unsigned idx
)
475 return 0x3c00u
; /* 1.0 */
477 return 0x3f800000u
; /* 1.0 */
479 return idx
? 0x3ff00000u
: 0u; /* 1.0 */
487 return idx
? 0x7fffffffu
: 0xffffffffu
;
495 return idx
? 0x80000000u
: 0;
507 return 0x7c00u
; /* infinity */
509 return 0x7f800000u
; /* infinity */
511 return idx
? 0x7ff00000u
: 0u; /* infinity */
513 return 0xfc00u
; /* negative infinity */
515 return 0xff800000u
; /* negative infinity */
517 return idx
? 0xfff00000u
: 0u; /* negative infinity */
519 unreachable("Invalid reduction operation");
525 void emit_ds_swizzle(Builder bld
, PhysReg dst
, PhysReg src
, unsigned size
, unsigned ds_pattern
)
527 for (unsigned i
= 0; i
< size
; i
++) {
528 bld
.ds(aco_opcode::ds_swizzle_b32
, Definition(PhysReg
{dst
+i
}, v1
),
529 Operand(PhysReg
{src
+i
}, v1
), ds_pattern
);
533 void emit_reduction(lower_context
*ctx
, aco_opcode op
, ReduceOp reduce_op
, unsigned cluster_size
, PhysReg tmp
,
534 PhysReg stmp
, PhysReg vtmp
, PhysReg sitmp
, Operand src
, Definition dst
)
536 assert(cluster_size
== ctx
->program
->wave_size
|| op
== aco_opcode::p_reduce
);
537 assert(cluster_size
<= ctx
->program
->wave_size
);
539 Builder
bld(ctx
->program
, &ctx
->instructions
);
542 identity
[0] = Operand(get_reduction_identity(reduce_op
, 0));
543 identity
[1] = Operand(get_reduction_identity(reduce_op
, 1));
544 Operand vcndmask_identity
[2] = {identity
[0], identity
[1]};
546 /* First, copy the source to tmp and set inactive lanes to the identity */
547 bld
.sop1(Builder::s_or_saveexec
, Definition(stmp
, bld
.lm
), Definition(scc
, s1
), Definition(exec
, bld
.lm
), Operand(UINT64_MAX
), Operand(exec
, bld
.lm
));
549 for (unsigned i
= 0; i
< src
.size(); i
++) {
550 /* p_exclusive_scan needs it to be a sgpr or inline constant for the v_writelane_b32
551 * except on GFX10, where v_writelane_b32 can take a literal. */
552 if (identity
[i
].isLiteral() && op
== aco_opcode::p_exclusive_scan
&& ctx
->program
->chip_class
< GFX10
) {
553 bld
.sop1(aco_opcode::s_mov_b32
, Definition(PhysReg
{sitmp
+i
}, s1
), identity
[i
]);
554 identity
[i
] = Operand(PhysReg
{sitmp
+i
}, s1
);
556 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{tmp
+i
}, v1
), identity
[i
]);
557 vcndmask_identity
[i
] = Operand(PhysReg
{tmp
+i
}, v1
);
558 } else if (identity
[i
].isLiteral()) {
559 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{tmp
+i
}, v1
), identity
[i
]);
560 vcndmask_identity
[i
] = Operand(PhysReg
{tmp
+i
}, v1
);
564 for (unsigned i
= 0; i
< src
.size(); i
++) {
565 bld
.vop2_e64(aco_opcode::v_cndmask_b32
, Definition(PhysReg
{tmp
+ i
}, v1
),
566 vcndmask_identity
[i
], Operand(PhysReg
{src
.physReg() + i
}, v1
),
567 Operand(stmp
, bld
.lm
));
570 if (src
.regClass() == v1b
) {
571 if (ctx
->program
->chip_class
>= GFX8
) {
572 aco_ptr
<SDWA_instruction
> sdwa
{create_instruction
<SDWA_instruction
>(aco_opcode::v_mov_b32
, asSDWA(Format::VOP1
), 1, 1)};
573 sdwa
->operands
[0] = Operand(PhysReg
{tmp
}, v1
);
574 sdwa
->definitions
[0] = Definition(PhysReg
{tmp
}, v1
);
575 if (reduce_op
== imin8
|| reduce_op
== imax8
)
576 sdwa
->sel
[0] = sdwa_sbyte
;
578 sdwa
->sel
[0] = sdwa_ubyte
;
579 sdwa
->dst_sel
= sdwa_udword
;
580 bld
.insert(std::move(sdwa
));
584 if (reduce_op
== imin8
|| reduce_op
== imax8
)
585 opcode
= aco_opcode::v_bfe_i32
;
587 opcode
= aco_opcode::v_bfe_u32
;
589 bld
.vop3(opcode
, Definition(PhysReg
{tmp
}, v1
),
590 Operand(PhysReg
{tmp
}, v1
), Operand(0u), Operand(8u));
592 } else if (src
.regClass() == v2b
) {
593 if (ctx
->program
->chip_class
>= GFX10
&&
594 (reduce_op
== iadd16
|| reduce_op
== imax16
||
595 reduce_op
== imin16
|| reduce_op
== umin16
|| reduce_op
== umax16
)) {
596 aco_ptr
<SDWA_instruction
> sdwa
{create_instruction
<SDWA_instruction
>(aco_opcode::v_mov_b32
, asSDWA(Format::VOP1
), 1, 1)};
597 sdwa
->operands
[0] = Operand(PhysReg
{tmp
}, v1
);
598 sdwa
->definitions
[0] = Definition(PhysReg
{tmp
}, v1
);
599 if (reduce_op
== imin16
|| reduce_op
== imax16
|| reduce_op
== iadd16
)
600 sdwa
->sel
[0] = sdwa_sword
;
602 sdwa
->sel
[0] = sdwa_uword
;
603 sdwa
->dst_sel
= sdwa_udword
;
604 bld
.insert(std::move(sdwa
));
605 } else if (ctx
->program
->chip_class
== GFX6
|| ctx
->program
->chip_class
== GFX7
) {
608 if (reduce_op
== imin16
|| reduce_op
== imax16
|| reduce_op
== iadd16
)
609 opcode
= aco_opcode::v_bfe_i32
;
611 opcode
= aco_opcode::v_bfe_u32
;
613 bld
.vop3(opcode
, Definition(PhysReg
{tmp
}, v1
),
614 Operand(PhysReg
{tmp
}, v1
), Operand(0u), Operand(16u));
618 bool reduction_needs_last_op
= false;
620 case aco_opcode::p_reduce
:
621 if (cluster_size
== 1) break;
623 if (ctx
->program
->chip_class
<= GFX7
) {
624 reduction_needs_last_op
= true;
625 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), (1 << 15) | dpp_quad_perm(1, 0, 3, 2));
626 if (cluster_size
== 2) break;
627 emit_op(ctx
, tmp
, vtmp
, tmp
, PhysReg
{0}, reduce_op
, src
.size());
628 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), (1 << 15) | dpp_quad_perm(2, 3, 0, 1));
629 if (cluster_size
== 4) break;
630 emit_op(ctx
, tmp
, vtmp
, tmp
, PhysReg
{0}, reduce_op
, src
.size());
631 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1f, 0, 0x04));
632 if (cluster_size
== 8) break;
633 emit_op(ctx
, tmp
, vtmp
, tmp
, PhysReg
{0}, reduce_op
, src
.size());
634 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1f, 0, 0x08));
635 if (cluster_size
== 16) break;
636 emit_op(ctx
, tmp
, vtmp
, tmp
, PhysReg
{0}, reduce_op
, src
.size());
637 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1f, 0, 0x10));
638 if (cluster_size
== 32) break;
639 emit_op(ctx
, tmp
, vtmp
, tmp
, PhysReg
{0}, reduce_op
, src
.size());
640 for (unsigned i
= 0; i
< src
.size(); i
++)
641 bld
.readlane(Definition(PhysReg
{dst
.physReg() + i
}, s1
), Operand(PhysReg
{tmp
+ i
}, v1
), Operand(0u));
642 // TODO: it would be more effective to do the last reduction step on SALU
643 emit_op(ctx
, tmp
, dst
.physReg(), tmp
, vtmp
, reduce_op
, src
.size());
644 reduction_needs_last_op
= false;
648 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(), dpp_quad_perm(1, 0, 3, 2), 0xf, 0xf, false);
649 if (cluster_size
== 2) break;
650 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(), dpp_quad_perm(2, 3, 0, 1), 0xf, 0xf, false);
651 if (cluster_size
== 4) break;
652 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(), dpp_row_half_mirror
, 0xf, 0xf, false);
653 if (cluster_size
== 8) break;
654 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(), dpp_row_mirror
, 0xf, 0xf, false);
655 if (cluster_size
== 16) break;
657 if (ctx
->program
->chip_class
>= GFX10
) {
658 /* GFX10+ doesn't support row_bcast15 and row_bcast31 */
659 for (unsigned i
= 0; i
< src
.size(); i
++)
660 bld
.vop3(aco_opcode::v_permlanex16_b32
, Definition(PhysReg
{vtmp
+i
}, v1
), Operand(PhysReg
{tmp
+i
}, v1
), Operand(0u), Operand(0u));
662 if (cluster_size
== 32) {
663 reduction_needs_last_op
= true;
667 emit_op(ctx
, tmp
, tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
668 for (unsigned i
= 0; i
< src
.size(); i
++)
669 bld
.readlane(Definition(PhysReg
{dst
.physReg() + i
}, s1
), Operand(PhysReg
{tmp
+i
}, v1
), Operand(0u));
670 // TODO: it would be more effective to do the last reduction step on SALU
671 emit_op(ctx
, tmp
, dst
.physReg(), tmp
, vtmp
, reduce_op
, src
.size());
675 if (cluster_size
== 32) {
676 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1f, 0, 0x10));
677 reduction_needs_last_op
= true;
680 assert(cluster_size
== 64);
681 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(), dpp_row_bcast15
, 0xa, 0xf, false);
682 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(), dpp_row_bcast31
, 0xc, 0xf, false);
684 case aco_opcode::p_exclusive_scan
:
685 if (ctx
->program
->chip_class
>= GFX10
) { /* gfx10 doesn't support wf_sr1, so emulate it */
686 /* shift rows right */
687 emit_dpp_mov(ctx
, vtmp
, tmp
, src
.size(), dpp_row_sr(1), 0xf, 0xf, true);
689 /* fill in the gaps in rows 1 and 3 */
690 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_lo
, s1
), Operand(0x10000u
));
691 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_hi
, s1
), Operand(0x10000u
));
692 for (unsigned i
= 0; i
< src
.size(); i
++) {
693 Instruction
*perm
= bld
.vop3(aco_opcode::v_permlanex16_b32
,
694 Definition(PhysReg
{vtmp
+i
}, v1
),
695 Operand(PhysReg
{tmp
+i
}, v1
),
696 Operand(0xffffffffu
), Operand(0xffffffffu
)).instr
;
697 static_cast<VOP3A_instruction
*>(perm
)->opsel
= 1; /* FI (Fetch Inactive) */
699 bld
.sop1(Builder::s_mov
, Definition(exec
, bld
.lm
), Operand(UINT64_MAX
));
701 if (ctx
->program
->wave_size
== 64) {
702 /* fill in the gap in row 2 */
703 for (unsigned i
= 0; i
< src
.size(); i
++) {
704 bld
.readlane(Definition(PhysReg
{sitmp
+i
}, s1
), Operand(PhysReg
{tmp
+i
}, v1
), Operand(31u));
705 bld
.writelane(Definition(PhysReg
{vtmp
+i
}, v1
), Operand(PhysReg
{sitmp
+i
}, s1
), Operand(32u), Operand(PhysReg
{vtmp
+i
}, v1
));
708 std::swap(tmp
, vtmp
);
709 } else if (ctx
->program
->chip_class
>= GFX8
) {
710 emit_dpp_mov(ctx
, tmp
, tmp
, src
.size(), dpp_wf_sr1
, 0xf, 0xf, true);
712 // TODO: use LDS on CS with a single write and shifted read
713 /* wavefront shift_right by 1 on SI/CI */
714 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), (1 << 15) | dpp_quad_perm(0, 0, 1, 2));
715 emit_ds_swizzle(bld
, tmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1F, 0x00, 0x07)); /* mirror(8) */
716 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_lo
, s1
), Operand(0x10101010u
));
717 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_hi
, s1
), Operand(exec_lo
, s1
));
718 for (unsigned i
= 0; i
< src
.size(); i
++)
719 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{vtmp
+i
}, v1
), Operand(PhysReg
{tmp
+i
}, v1
));
721 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(UINT64_MAX
));
722 emit_ds_swizzle(bld
, tmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1F, 0x00, 0x08)); /* swap(8) */
723 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_lo
, s1
), Operand(0x01000100u
));
724 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_hi
, s1
), Operand(exec_lo
, s1
));
725 for (unsigned i
= 0; i
< src
.size(); i
++)
726 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{vtmp
+i
}, v1
), Operand(PhysReg
{tmp
+i
}, v1
));
728 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(UINT64_MAX
));
729 emit_ds_swizzle(bld
, tmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1F, 0x00, 0x10)); /* swap(16) */
730 bld
.sop2(aco_opcode::s_bfm_b32
, Definition(exec_lo
, s1
), Operand(1u), Operand(16u));
731 bld
.sop2(aco_opcode::s_bfm_b32
, Definition(exec_hi
, s1
), Operand(1u), Operand(16u));
732 for (unsigned i
= 0; i
< src
.size(); i
++)
733 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{vtmp
+i
}, v1
), Operand(PhysReg
{tmp
+i
}, v1
));
735 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(UINT64_MAX
));
736 for (unsigned i
= 0; i
< src
.size(); i
++) {
737 bld
.writelane(Definition(PhysReg
{vtmp
+i
}, v1
), identity
[i
], Operand(0u), Operand(PhysReg
{vtmp
+i
}, v1
));
738 bld
.readlane(Definition(PhysReg
{sitmp
+i
}, s1
), Operand(PhysReg
{tmp
+i
}, v1
), Operand(0u));
739 bld
.writelane(Definition(PhysReg
{vtmp
+i
}, v1
), Operand(PhysReg
{sitmp
+i
}, s1
), Operand(32u), Operand(PhysReg
{vtmp
+i
}, v1
));
740 identity
[i
] = Operand(0u); /* prevent further uses of identity */
742 std::swap(tmp
, vtmp
);
745 for (unsigned i
= 0; i
< src
.size(); i
++) {
746 if (!identity
[i
].isConstant() || identity
[i
].constantValue()) { /* bound_ctrl should take care of this overwise */
747 if (ctx
->program
->chip_class
< GFX10
)
748 assert((identity
[i
].isConstant() && !identity
[i
].isLiteral()) || identity
[i
].physReg() == PhysReg
{sitmp
+i
});
749 bld
.writelane(Definition(PhysReg
{tmp
+i
}, v1
), identity
[i
], Operand(0u), Operand(PhysReg
{tmp
+i
}, v1
));
753 case aco_opcode::p_inclusive_scan
:
754 assert(cluster_size
== ctx
->program
->wave_size
);
755 if (ctx
->program
->chip_class
<= GFX7
) {
756 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1e, 0x00, 0x00));
757 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_lo
, s1
), Operand(0xAAAAAAAAu
));
758 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_hi
, s1
), Operand(exec_lo
, s1
));
759 emit_op(ctx
, tmp
, tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
761 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(UINT64_MAX
));
762 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1c, 0x01, 0x00));
763 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_lo
, s1
), Operand(0xCCCCCCCCu
));
764 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_hi
, s1
), Operand(exec_lo
, s1
));
765 emit_op(ctx
, tmp
, tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
767 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(UINT64_MAX
));
768 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x18, 0x03, 0x00));
769 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_lo
, s1
), Operand(0xF0F0F0F0u
));
770 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_hi
, s1
), Operand(exec_lo
, s1
));
771 emit_op(ctx
, tmp
, tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
773 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(UINT64_MAX
));
774 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x10, 0x07, 0x00));
775 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_lo
, s1
), Operand(0xFF00FF00u
));
776 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_hi
, s1
), Operand(exec_lo
, s1
));
777 emit_op(ctx
, tmp
, tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
779 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(UINT64_MAX
));
780 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x00, 0x0f, 0x00));
781 bld
.sop2(aco_opcode::s_bfm_b32
, Definition(exec_lo
, s1
), Operand(16u), Operand(16u));
782 bld
.sop2(aco_opcode::s_bfm_b32
, Definition(exec_hi
, s1
), Operand(16u), Operand(16u));
783 emit_op(ctx
, tmp
, tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
785 for (unsigned i
= 0; i
< src
.size(); i
++)
786 bld
.readlane(Definition(PhysReg
{sitmp
+i
}, s1
), Operand(PhysReg
{tmp
+i
}, v1
), Operand(31u));
787 bld
.sop2(aco_opcode::s_bfm_b64
, Definition(exec
, s2
), Operand(32u), Operand(32u));
788 emit_op(ctx
, tmp
, sitmp
, tmp
, vtmp
, reduce_op
, src
.size());
792 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(),
793 dpp_row_sr(1), 0xf, 0xf, false, identity
);
794 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(),
795 dpp_row_sr(2), 0xf, 0xf, false, identity
);
796 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(),
797 dpp_row_sr(4), 0xf, 0xf, false, identity
);
798 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(),
799 dpp_row_sr(8), 0xf, 0xf, false, identity
);
800 if (ctx
->program
->chip_class
>= GFX10
) {
801 bld
.sop2(aco_opcode::s_bfm_b32
, Definition(exec_lo
, s1
), Operand(16u), Operand(16u));
802 bld
.sop2(aco_opcode::s_bfm_b32
, Definition(exec_hi
, s1
), Operand(16u), Operand(16u));
803 for (unsigned i
= 0; i
< src
.size(); i
++) {
804 Instruction
*perm
= bld
.vop3(aco_opcode::v_permlanex16_b32
,
805 Definition(PhysReg
{vtmp
+i
}, v1
),
806 Operand(PhysReg
{tmp
+i
}, v1
),
807 Operand(0xffffffffu
), Operand(0xffffffffu
)).instr
;
808 static_cast<VOP3A_instruction
*>(perm
)->opsel
= 1; /* FI (Fetch Inactive) */
810 emit_op(ctx
, tmp
, tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
812 if (ctx
->program
->wave_size
== 64) {
813 bld
.sop2(aco_opcode::s_bfm_b64
, Definition(exec
, s2
), Operand(32u), Operand(32u));
814 for (unsigned i
= 0; i
< src
.size(); i
++)
815 bld
.readlane(Definition(PhysReg
{sitmp
+i
}, s1
), Operand(PhysReg
{tmp
+i
}, v1
), Operand(31u));
816 emit_op(ctx
, tmp
, sitmp
, tmp
, vtmp
, reduce_op
, src
.size());
819 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(),
820 dpp_row_bcast15
, 0xa, 0xf, false, identity
);
821 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(),
822 dpp_row_bcast31
, 0xc, 0xf, false, identity
);
826 unreachable("Invalid reduction mode");
830 if (op
== aco_opcode::p_reduce
) {
831 if (reduction_needs_last_op
&& dst
.regClass().type() == RegType::vgpr
) {
832 bld
.sop1(Builder::s_mov
, Definition(exec
, bld
.lm
), Operand(stmp
, bld
.lm
));
833 emit_op(ctx
, dst
.physReg(), tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
837 if (reduction_needs_last_op
)
838 emit_op(ctx
, tmp
, vtmp
, tmp
, PhysReg
{0}, reduce_op
, src
.size());
842 bld
.sop1(Builder::s_mov
, Definition(exec
, bld
.lm
), Operand(stmp
, bld
.lm
));
844 if (dst
.regClass().type() == RegType::sgpr
) {
845 for (unsigned k
= 0; k
< src
.size(); k
++) {
846 bld
.readlane(Definition(PhysReg
{dst
.physReg() + k
}, s1
),
847 Operand(PhysReg
{tmp
+ k
}, v1
), Operand(ctx
->program
->wave_size
- 1));
849 } else if (dst
.physReg() != tmp
) {
850 for (unsigned k
= 0; k
< src
.size(); k
++) {
851 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{dst
.physReg() + k
}, v1
),
852 Operand(PhysReg
{tmp
+ k
}, v1
));
857 void emit_gfx10_wave64_bpermute(Program
*program
, aco_ptr
<Instruction
> &instr
, Builder
&bld
)
859 /* Emulates proper bpermute on GFX10 in wave64 mode.
861 * This is necessary because on GFX10 the bpermute instruction only works
862 * on half waves (you can think of it as having a cluster size of 32), so we
863 * manually swap the data between the two halves using two shared VGPRs.
866 assert(program
->chip_class
>= GFX10
);
867 assert(program
->info
->wave_size
== 64);
869 unsigned shared_vgpr_reg_0
= align(program
->config
->num_vgprs
, 4) + 256;
870 Definition dst
= instr
->definitions
[0];
871 Definition tmp_exec
= instr
->definitions
[1];
872 Definition clobber_scc
= instr
->definitions
[2];
873 Operand index_x4
= instr
->operands
[0];
874 Operand input_data
= instr
->operands
[1];
875 Operand same_half
= instr
->operands
[2];
877 assert(dst
.regClass() == v1
);
878 assert(tmp_exec
.regClass() == bld
.lm
);
879 assert(clobber_scc
.isFixed() && clobber_scc
.physReg() == scc
);
880 assert(same_half
.regClass() == bld
.lm
);
881 assert(index_x4
.regClass() == v1
);
882 assert(input_data
.regClass().type() == RegType::vgpr
);
883 assert(input_data
.bytes() <= 4);
884 assert(dst
.physReg() != index_x4
.physReg());
885 assert(dst
.physReg() != input_data
.physReg());
886 assert(tmp_exec
.physReg() != same_half
.physReg());
888 PhysReg
shared_vgpr_lo(shared_vgpr_reg_0
);
889 PhysReg
shared_vgpr_hi(shared_vgpr_reg_0
+ 1);
891 /* Permute the input within the same half-wave */
892 bld
.ds(aco_opcode::ds_bpermute_b32
, dst
, index_x4
, input_data
);
894 /* HI: Copy data from high lanes 32-63 to shared vgpr */
895 bld
.vop1_dpp(aco_opcode::v_mov_b32
, Definition(shared_vgpr_hi
, v1
), input_data
, dpp_quad_perm(0, 1, 2, 3), 0xc, 0xf, false);
897 bld
.sop1(aco_opcode::s_mov_b64
, tmp_exec
, Operand(exec
, s2
));
898 /* Set EXEC to enable LO lanes only */
899 bld
.sop2(aco_opcode::s_bfm_b64
, Definition(exec
, s2
), Operand(32u), Operand(0u));
900 /* LO: Copy data from low lanes 0-31 to shared vgpr */
901 bld
.vop1(aco_opcode::v_mov_b32
, Definition(shared_vgpr_lo
, v1
), input_data
);
902 /* LO: bpermute shared vgpr (high lanes' data) */
903 bld
.ds(aco_opcode::ds_bpermute_b32
, Definition(shared_vgpr_hi
, v1
), index_x4
, Operand(shared_vgpr_hi
, v1
));
904 /* Set EXEC to enable HI lanes only */
905 bld
.sop2(aco_opcode::s_bfm_b64
, Definition(exec
, s2
), Operand(32u), Operand(32u));
906 /* HI: bpermute shared vgpr (low lanes' data) */
907 bld
.ds(aco_opcode::ds_bpermute_b32
, Definition(shared_vgpr_lo
, v1
), index_x4
, Operand(shared_vgpr_lo
, v1
));
909 /* Only enable lanes which use the other half's data */
910 bld
.sop2(aco_opcode::s_andn2_b64
, Definition(exec
, s2
), clobber_scc
, Operand(tmp_exec
.physReg(), s2
), same_half
);
911 /* LO: Copy shared vgpr (high lanes' bpermuted data) to output vgpr */
912 bld
.vop1_dpp(aco_opcode::v_mov_b32
, dst
, Operand(shared_vgpr_hi
, v1
), dpp_quad_perm(0, 1, 2, 3), 0x3, 0xf, false);
913 /* HI: Copy shared vgpr (low lanes' bpermuted data) to output vgpr */
914 bld
.vop1_dpp(aco_opcode::v_mov_b32
, dst
, Operand(shared_vgpr_lo
, v1
), dpp_quad_perm(0, 1, 2, 3), 0xc, 0xf, false);
916 /* Restore saved EXEC */
917 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(tmp_exec
.physReg(), s2
));
919 /* RA assumes that the result is always in the low part of the register, so we have to shift, if it's not there already */
920 if (input_data
.physReg().byte()) {
921 unsigned right_shift
= input_data
.physReg().byte() * 8;
922 bld
.vop2(aco_opcode::v_lshrrev_b32
, dst
, Operand(right_shift
), Operand(dst
.physReg(), v1
));
926 void emit_gfx6_bpermute(Program
*program
, aco_ptr
<Instruction
> &instr
, Builder
&bld
)
928 /* Emulates bpermute using readlane instructions */
930 Operand index
= instr
->operands
[0];
931 Operand input
= instr
->operands
[1];
932 Definition dst
= instr
->definitions
[0];
933 Definition temp_exec
= instr
->definitions
[1];
934 Definition clobber_vcc
= instr
->definitions
[2];
936 assert(dst
.regClass() == v1
);
937 assert(temp_exec
.regClass() == bld
.lm
);
938 assert(clobber_vcc
.regClass() == bld
.lm
);
939 assert(clobber_vcc
.physReg() == vcc
);
940 assert(index
.regClass() == v1
);
941 assert(index
.physReg() != dst
.physReg());
942 assert(input
.regClass().type() == RegType::vgpr
);
943 assert(input
.bytes() <= 4);
944 assert(input
.physReg() != dst
.physReg());
946 /* Save original EXEC */
947 bld
.sop1(aco_opcode::s_mov_b64
, temp_exec
, Operand(exec
, s2
));
949 /* An "unrolled loop" that is executed per each lane.
950 * This takes only a few instructions per lane, as opposed to a "real" loop
951 * with branching, where the branch instruction alone would take 16+ cycles.
953 for (unsigned n
= 0; n
< program
->wave_size
; ++n
) {
954 /* Activate the lane which has N for its source index */
955 bld
.vopc(aco_opcode::v_cmpx_eq_u32
, Definition(exec
, bld
.lm
), clobber_vcc
, Operand(n
), index
);
956 /* Read the data from lane N */
957 bld
.readlane(Definition(vcc
, s1
), input
, Operand(n
));
958 /* On the active lane, move the data we read from lane N to the destination VGPR */
959 bld
.vop1(aco_opcode::v_mov_b32
, dst
, Operand(vcc
, s1
));
960 /* Restore original EXEC */
961 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(temp_exec
.physReg(), s2
));
965 struct copy_operation
{
971 uint64_t is_used
= 0;
975 void split_copy(unsigned offset
, Definition
*def
, Operand
*op
, const copy_operation
& src
, bool ignore_uses
, unsigned max_size
)
977 PhysReg def_reg
= src
.def
.physReg();
978 PhysReg op_reg
= src
.op
.physReg();
979 def_reg
.reg_b
+= offset
;
980 op_reg
.reg_b
+= offset
;
982 max_size
= MIN2(max_size
, src
.def
.regClass().type() == RegType::vgpr
? 4 : 8);
984 /* make sure the size is a power of two and reg % bytes == 0 */
986 for (; bytes
<= max_size
; bytes
*= 2) {
987 unsigned next
= bytes
* 2u;
988 bool can_increase
= def_reg
.reg_b
% next
== 0 &&
989 offset
+ next
<= src
.bytes
&& next
<= max_size
;
990 if (!src
.op
.isConstant() && can_increase
)
991 can_increase
= op_reg
.reg_b
% next
== 0;
992 for (unsigned i
= 0; !ignore_uses
&& can_increase
&& (i
< bytes
); i
++)
993 can_increase
= (src
.uses
[offset
+ bytes
+ i
] == 0) == (src
.uses
[offset
] == 0);
998 RegClass def_cls
= bytes
% 4 == 0 ? RegClass(src
.def
.regClass().type(), bytes
/ 4u) :
999 RegClass(src
.def
.regClass().type(), bytes
).as_subdword();
1000 *def
= Definition(src
.def
.tempId(), def_reg
, def_cls
);
1001 if (src
.op
.isConstant()) {
1002 assert(offset
== 0 || (offset
== 4 && src
.op
.bytes() == 8));
1003 if (src
.op
.bytes() == 8 && bytes
== 4)
1004 *op
= Operand(uint32_t(src
.op
.constantValue64() >> (offset
* 8u)));
1008 RegClass op_cls
= bytes
% 4 == 0 ? RegClass(src
.op
.regClass().type(), bytes
/ 4u) :
1009 RegClass(src
.op
.regClass().type(), bytes
).as_subdword();
1010 *op
= Operand(op_reg
, op_cls
);
1011 op
->setTemp(Temp(src
.op
.tempId(), op_cls
));
1015 uint32_t get_intersection_mask(int a_start
, int a_size
,
1016 int b_start
, int b_size
)
1018 int intersection_start
= MAX2(b_start
- a_start
, 0);
1019 int intersection_end
= MAX2(b_start
+ b_size
- a_start
, 0);
1020 if (intersection_start
>= a_size
|| intersection_end
== 0)
1023 uint32_t mask
= u_bit_consecutive(0, a_size
);
1024 return u_bit_consecutive(intersection_start
, intersection_end
- intersection_start
) & mask
;
1027 bool do_copy(lower_context
* ctx
, Builder
& bld
, const copy_operation
& copy
, bool *preserve_scc
)
1029 bool did_copy
= false;
1030 for (unsigned offset
= 0; offset
< copy
.bytes
;) {
1031 if (copy
.uses
[offset
]) {
1038 split_copy(offset
, &def
, &op
, copy
, false, 8);
1040 if (def
.physReg() == scc
) {
1041 bld
.sopc(aco_opcode::s_cmp_lg_i32
, def
, op
, Operand(0u));
1042 *preserve_scc
= true;
1043 } else if (def
.bytes() == 8 && def
.getTemp().type() == RegType::sgpr
) {
1044 bld
.sop1(aco_opcode::s_mov_b64
, def
, Operand(op
.physReg(), s2
));
1045 } else if (def
.regClass().is_subdword() && ctx
->program
->chip_class
< GFX8
) {
1046 if (op
.physReg().byte()) {
1047 assert(def
.physReg().byte() == 0);
1048 bld
.vop2(aco_opcode::v_lshrrev_b32
, def
, Operand(op
.physReg().byte() * 8), op
);
1049 } else if (def
.physReg().byte() == 2) {
1050 assert(op
.physReg().byte() == 0);
1051 /* preserve the target's lower half */
1052 def
= Definition(def
.physReg().advance(-2), v1
);
1053 bld
.vop2(aco_opcode::v_and_b32
, Definition(op
.physReg(), v1
), Operand(0xFFFFu
), op
);
1054 if (def
.physReg().reg() != op
.physReg().reg())
1055 bld
.vop2(aco_opcode::v_and_b32
, def
, Operand(0xFFFFu
), Operand(def
.physReg(), v2b
));
1056 bld
.vop2(aco_opcode::v_cvt_pk_u16_u32
, def
, Operand(def
.physReg(), v2b
), op
);
1057 } else if (def
.physReg().byte()) {
1058 unsigned bits
= def
.physReg().byte() * 8;
1059 assert(op
.physReg().byte() == 0);
1060 def
= Definition(def
.physReg().advance(-def
.physReg().byte()), v1
);
1061 bld
.vop2(aco_opcode::v_and_b32
, def
, Operand((1 << bits
) - 1u), Operand(def
.physReg(), op
.regClass()));
1062 bld
.vop2(aco_opcode::v_lshlrev_b32
, Definition(op
.physReg(), def
.regClass()), Operand(bits
), op
);
1063 bld
.vop2(aco_opcode::v_or_b32
, def
, Operand(def
.physReg(), op
.regClass()), op
);
1064 bld
.vop2(aco_opcode::v_lshrrev_b32
, Definition(op
.physReg(), def
.regClass()), Operand(bits
), op
);
1066 bld
.vop1(aco_opcode::v_mov_b32
, def
, op
);
1073 offset
+= def
.bytes();
1078 void do_swap(lower_context
*ctx
, Builder
& bld
, const copy_operation
& copy
, bool preserve_scc
, Pseudo_instruction
*pi
)
1080 unsigned offset
= 0;
1082 if (copy
.bytes
== 3 && (copy
.def
.physReg().reg_b
% 4 <= 1) &&
1083 (copy
.def
.physReg().reg_b
% 4) == (copy
.op
.physReg().reg_b
% 4)) {
1084 /* instead of doing a 2-byte and 1-byte swap, do a 4-byte swap and then fixup with a 1-byte swap */
1085 PhysReg op
= copy
.op
.physReg();
1086 PhysReg def
= copy
.def
.physReg();
1091 tmp
.op
= Operand(op
, v1
);
1092 tmp
.def
= Definition(def
, v1
);
1094 memset(tmp
.uses
, 1, 4);
1095 do_swap(ctx
, bld
, tmp
, preserve_scc
, pi
);
1097 op
.reg_b
+= copy
.def
.physReg().reg_b
% 4 == 0 ? 3 : 0;
1098 def
.reg_b
+= copy
.def
.physReg().reg_b
% 4 == 0 ? 3 : 0;
1099 tmp
.op
= Operand(op
, v1b
);
1100 tmp
.def
= Definition(def
, v1b
);
1103 do_swap(ctx
, bld
, tmp
, preserve_scc
, pi
);
1105 offset
= copy
.bytes
;
1108 for (; offset
< copy
.bytes
;) {
1111 split_copy(offset
, &def
, &op
, copy
, true, 8);
1113 assert(op
.regClass() == def
.regClass());
1114 Operand def_as_op
= Operand(def
.physReg(), def
.regClass());
1115 Definition op_as_def
= Definition(op
.physReg(), op
.regClass());
1116 if (ctx
->program
->chip_class
>= GFX9
&& def
.regClass() == v1
) {
1117 bld
.vop1(aco_opcode::v_swap_b32
, def
, op_as_def
, op
, def_as_op
);
1118 } else if (def
.regClass() == v1
|| (def
.regClass().is_subdword() && ctx
->program
->chip_class
< GFX8
)) {
1119 assert(def
.physReg().byte() == 0 && op
.physReg().byte() == 0);
1120 bld
.vop2(aco_opcode::v_xor_b32
, op_as_def
, op
, def_as_op
);
1121 bld
.vop2(aco_opcode::v_xor_b32
, def
, op
, def_as_op
);
1122 bld
.vop2(aco_opcode::v_xor_b32
, op_as_def
, op
, def_as_op
);
1123 } else if (op
.physReg() == scc
|| def
.physReg() == scc
) {
1124 /* we need to swap scc and another sgpr */
1125 assert(!preserve_scc
);
1127 PhysReg other
= op
.physReg() == scc
? def
.physReg() : op
.physReg();
1129 bld
.sop1(aco_opcode::s_mov_b32
, Definition(pi
->scratch_sgpr
, s1
), Operand(scc
, s1
));
1130 bld
.sopc(aco_opcode::s_cmp_lg_i32
, Definition(scc
, s1
), Operand(other
, s1
), Operand(0u));
1131 bld
.sop1(aco_opcode::s_mov_b32
, Definition(other
, s1
), Operand(pi
->scratch_sgpr
, s1
));
1132 } else if (def
.regClass() == s1
) {
1134 bld
.sop1(aco_opcode::s_mov_b32
, Definition(pi
->scratch_sgpr
, s1
), op
);
1135 bld
.sop1(aco_opcode::s_mov_b32
, op_as_def
, def_as_op
);
1136 bld
.sop1(aco_opcode::s_mov_b32
, def
, Operand(pi
->scratch_sgpr
, s1
));
1138 bld
.sop2(aco_opcode::s_xor_b32
, op_as_def
, Definition(scc
, s1
), op
, def_as_op
);
1139 bld
.sop2(aco_opcode::s_xor_b32
, def
, Definition(scc
, s1
), op
, def_as_op
);
1140 bld
.sop2(aco_opcode::s_xor_b32
, op_as_def
, Definition(scc
, s1
), op
, def_as_op
);
1142 } else if (def
.regClass() == s2
) {
1144 bld
.sop1(aco_opcode::s_mov_b32
, Definition(pi
->scratch_sgpr
, s1
), Operand(scc
, s1
));
1145 bld
.sop2(aco_opcode::s_xor_b64
, op_as_def
, Definition(scc
, s1
), op
, def_as_op
);
1146 bld
.sop2(aco_opcode::s_xor_b64
, def
, Definition(scc
, s1
), op
, def_as_op
);
1147 bld
.sop2(aco_opcode::s_xor_b64
, op_as_def
, Definition(scc
, s1
), op
, def_as_op
);
1149 bld
.sopc(aco_opcode::s_cmp_lg_i32
, Definition(scc
, s1
), Operand(pi
->scratch_sgpr
, s1
), Operand(0u));
1150 } else if (ctx
->program
->chip_class
>= GFX9
&& def
.bytes() == 2 && def
.physReg().reg() == op
.physReg().reg()) {
1151 aco_ptr
<VOP3P_instruction
> vop3p
{create_instruction
<VOP3P_instruction
>(aco_opcode::v_pk_add_u16
, Format::VOP3P
, 2, 1)};
1152 vop3p
->operands
[0] = Operand(PhysReg
{op
.physReg().reg()}, v1
);
1153 vop3p
->operands
[1] = Operand(0u);
1154 vop3p
->definitions
[0] = Definition(PhysReg
{op
.physReg().reg()}, v1
);
1155 vop3p
->opsel_lo
= 0x1;
1156 vop3p
->opsel_hi
= 0x2;
1157 bld
.insert(std::move(vop3p
));
1159 assert(def
.regClass().is_subdword());
1160 bld
.vop2_sdwa(aco_opcode::v_xor_b32
, op_as_def
, op
, def_as_op
);
1161 bld
.vop2_sdwa(aco_opcode::v_xor_b32
, def
, op
, def_as_op
);
1162 bld
.vop2_sdwa(aco_opcode::v_xor_b32
, op_as_def
, op
, def_as_op
);
1165 offset
+= def
.bytes();
1168 if (ctx
->program
->chip_class
<= GFX7
)
1171 /* fixup in case we swapped bytes we shouldn't have */
1172 copy_operation tmp_copy
= copy
;
1173 tmp_copy
.op
.setFixed(copy
.def
.physReg());
1174 tmp_copy
.def
.setFixed(copy
.op
.physReg());
1175 do_copy(ctx
, bld
, tmp_copy
, &preserve_scc
);
1178 void handle_operands(std::map
<PhysReg
, copy_operation
>& copy_map
, lower_context
* ctx
, chip_class chip_class
, Pseudo_instruction
*pi
)
1180 Builder
bld(ctx
->program
, &ctx
->instructions
);
1181 unsigned num_instructions_before
= ctx
->instructions
.size();
1182 aco_ptr
<Instruction
> mov
;
1183 std::map
<PhysReg
, copy_operation
>::iterator it
= copy_map
.begin();
1184 std::map
<PhysReg
, copy_operation
>::iterator target
;
1185 bool writes_scc
= false;
1187 /* count the number of uses for each dst reg */
1188 while (it
!= copy_map
.end()) {
1190 if (it
->second
.def
.physReg() == scc
)
1193 assert(!pi
->tmp_in_scc
|| !(it
->second
.def
.physReg() == pi
->scratch_sgpr
));
1195 /* if src and dst reg are the same, remove operation */
1196 if (it
->first
== it
->second
.op
.physReg()) {
1197 it
= copy_map
.erase(it
);
1201 /* split large copies */
1202 if (it
->second
.bytes
> 8) {
1203 assert(!it
->second
.op
.isConstant());
1204 assert(!it
->second
.def
.regClass().is_subdword());
1205 RegClass rc
= RegClass(it
->second
.def
.regClass().type(), it
->second
.def
.size() - 2);
1206 Definition hi_def
= Definition(PhysReg
{it
->first
+ 2}, rc
);
1207 rc
= RegClass(it
->second
.op
.regClass().type(), it
->second
.op
.size() - 2);
1208 Operand hi_op
= Operand(PhysReg
{it
->second
.op
.physReg() + 2}, rc
);
1209 copy_operation copy
= {hi_op
, hi_def
, it
->second
.bytes
- 8};
1210 copy_map
[hi_def
.physReg()] = copy
;
1211 assert(it
->second
.op
.physReg().byte() == 0 && it
->second
.def
.physReg().byte() == 0);
1212 it
->second
.op
= Operand(it
->second
.op
.physReg(), it
->second
.op
.regClass().type() == RegType::sgpr
? s2
: v2
);
1213 it
->second
.def
= Definition(it
->second
.def
.physReg(), it
->second
.def
.regClass().type() == RegType::sgpr
? s2
: v2
);
1214 it
->second
.bytes
= 8;
1217 /* check if the definition reg is used by another copy operation */
1218 for (std::pair
<const PhysReg
, copy_operation
>& copy
: copy_map
) {
1219 if (copy
.second
.op
.isConstant())
1221 for (uint16_t i
= 0; i
< it
->second
.bytes
; i
++) {
1222 /* distance might underflow */
1223 unsigned distance
= it
->first
.reg_b
+ i
- copy
.second
.op
.physReg().reg_b
;
1224 if (distance
< copy
.second
.bytes
)
1225 it
->second
.uses
[i
] += 1;
1232 /* first, handle paths in the location transfer graph */
1233 bool preserve_scc
= pi
->tmp_in_scc
&& !writes_scc
;
1234 it
= copy_map
.begin();
1235 while (it
!= copy_map
.end()) {
1237 /* try to coalesce 32-bit sgpr copies to 64-bit copies */
1238 if (it
->second
.is_used
== 0 &&
1239 it
->second
.def
.getTemp().type() == RegType::sgpr
&& it
->second
.bytes
== 4 &&
1240 !it
->second
.op
.isConstant() && it
->first
% 2 == it
->second
.op
.physReg() % 2) {
1242 PhysReg other_def_reg
= PhysReg
{it
->first
% 2 ? it
->first
- 1 : it
->first
+ 1};
1243 PhysReg other_op_reg
= PhysReg
{it
->first
% 2 ? it
->second
.op
.physReg() - 1 : it
->second
.op
.physReg() + 1};
1244 std::map
<PhysReg
, copy_operation
>::iterator other
= copy_map
.find(other_def_reg
);
1246 if (other
!= copy_map
.end() && !other
->second
.is_used
&& other
->second
.bytes
== 4 &&
1247 other
->second
.op
.physReg() == other_op_reg
&& !other
->second
.op
.isConstant()) {
1248 std::map
<PhysReg
, copy_operation
>::iterator to_erase
= it
->first
% 2 ? it
: other
;
1249 it
= it
->first
% 2 ? other
: it
;
1250 copy_map
.erase(to_erase
);
1251 it
->second
.bytes
= 8;
1254 // TODO: try to coalesce subdword copies
1256 /* on GFX6/7, we need some small workarounds as there is no
1257 * SDWA instruction to do partial register writes */
1258 if (ctx
->program
->chip_class
< GFX8
&& it
->second
.bytes
< 4) {
1259 if (it
->first
.byte() == 0 && it
->second
.op
.physReg().byte() == 0 &&
1260 !it
->second
.is_used
&& pi
->opcode
== aco_opcode::p_split_vector
) {
1261 /* Other operations might overwrite the high bits, so change all users
1262 * of the high bits to the new target where they are still available.
1263 * This mechanism depends on also emitting dead definitions. */
1264 PhysReg reg_hi
= it
->second
.op
.physReg().advance(it
->second
.bytes
);
1265 while (reg_hi
!= PhysReg(it
->second
.op
.physReg().reg() + 1)) {
1266 std::map
<PhysReg
, copy_operation
>::iterator other
= copy_map
.begin();
1267 for (other
= copy_map
.begin(); other
!= copy_map
.end(); other
++) {
1268 /* on GFX6/7, if the high bits are used as operand, they cannot be a target */
1269 if (other
->second
.op
.physReg() == reg_hi
) {
1270 other
->second
.op
.setFixed(it
->first
.advance(reg_hi
.byte()));
1271 break; /* break because an operand can only be used once */
1274 reg_hi
= reg_hi
.advance(it
->second
.bytes
);
1276 } else if (it
->first
.byte()) {
1277 assert(pi
->opcode
== aco_opcode::p_create_vector
);
1278 /* on GFX6/7, if we target an upper half where the lower half hasn't yet been handled,
1279 * move to the target operand's high bits. This is save to do as it cannot be an operand */
1280 PhysReg lo
= PhysReg(it
->first
.reg());
1281 std::map
<PhysReg
, copy_operation
>::iterator other
= copy_map
.find(lo
);
1282 if (other
!= copy_map
.end()) {
1283 assert(other
->second
.bytes
== it
->first
.byte());
1284 PhysReg new_reg_hi
= other
->second
.op
.physReg().advance(it
->first
.byte());
1285 it
->second
.def
= Definition(new_reg_hi
, it
->second
.def
.regClass());
1286 it
->second
.is_used
= 0;
1287 other
->second
.bytes
+= it
->second
.bytes
;
1288 other
->second
.def
.setTemp(Temp(other
->second
.def
.tempId(), RegClass::get(RegType::vgpr
, other
->second
.bytes
)));
1289 other
->second
.op
.setTemp(Temp(other
->second
.op
.tempId(), RegClass::get(RegType::vgpr
, other
->second
.bytes
)));
1290 /* if the new target's high bits are also a target, change uses */
1291 std::map
<PhysReg
, copy_operation
>::iterator target
= copy_map
.find(new_reg_hi
);
1292 if (target
!= copy_map
.end()) {
1293 for (unsigned i
= 0; i
< it
->second
.bytes
; i
++)
1294 target
->second
.uses
[i
]++;
1300 /* find portions where the target reg is not used as operand for any other copy */
1301 if (it
->second
.is_used
) {
1302 if (it
->second
.op
.isConstant()) {
1303 /* we have to skip constants until is_used=0 */
1308 unsigned has_zero_use_bytes
= 0;
1309 for (unsigned i
= 0; i
< it
->second
.bytes
; i
++)
1310 has_zero_use_bytes
|= (it
->second
.uses
[i
] == 0) << i
;
1312 if (has_zero_use_bytes
) {
1313 /* Skipping partial copying and doing a v_swap_b32 and then fixup
1314 * copies is usually beneficial for sub-dword copies, but if doing
1315 * a partial copy allows further copies, it should be done instead. */
1316 bool partial_copy
= (has_zero_use_bytes
== 0xf) || (has_zero_use_bytes
== 0xf0);
1317 for (std::pair
<const PhysReg
, copy_operation
>& copy
: copy_map
) {
1320 for (uint16_t i
= 0; i
< copy
.second
.bytes
; i
++) {
1321 /* distance might underflow */
1322 unsigned distance
= copy
.first
.reg_b
+ i
- it
->second
.op
.physReg().reg_b
;
1323 if (distance
< it
->second
.bytes
&& copy
.second
.uses
[i
] == 1 &&
1324 !it
->second
.uses
[distance
])
1325 partial_copy
= true;
1329 if (!partial_copy
) {
1334 /* full target reg is used: register swapping needed */
1340 bool did_copy
= do_copy(ctx
, bld
, it
->second
, &preserve_scc
);
1342 std::pair
<PhysReg
, copy_operation
> copy
= *it
;
1344 if (it
->second
.is_used
== 0) {
1345 /* the target reg is not used as operand for any other copy, so we
1346 * copied to all of it */
1348 it
= copy_map
.begin();
1350 /* we only performed some portions of this copy, so split it to only
1351 * leave the portions that still need to be done */
1352 copy_operation original
= it
->second
; /* the map insertion below can overwrite this */
1354 for (unsigned offset
= 0; offset
< original
.bytes
;) {
1355 if (original
.uses
[offset
] == 0) {
1361 split_copy(offset
, &def
, &op
, original
, false, 8);
1363 copy_operation copy
= {op
, def
, def
.bytes()};
1364 for (unsigned i
= 0; i
< copy
.bytes
; i
++)
1365 copy
.uses
[i
] = original
.uses
[i
+ offset
];
1366 copy_map
[def
.physReg()] = copy
;
1368 offset
+= def
.bytes();
1371 it
= copy_map
.begin();
1374 /* Reduce the number of uses of the operand reg by one. Do this after
1375 * splitting the copy or removing it in case the copy writes to it's own
1376 * operand (for example, v[7:8] = v[8:9]) */
1377 if (did_copy
&& !copy
.second
.op
.isConstant()) {
1378 for (std::pair
<const PhysReg
, copy_operation
>& other
: copy_map
) {
1379 for (uint16_t i
= 0; i
< other
.second
.bytes
; i
++) {
1380 /* distance might underflow */
1381 unsigned distance
= other
.first
.reg_b
+ i
- copy
.second
.op
.physReg().reg_b
;
1382 if (distance
< copy
.second
.bytes
&& !copy
.second
.uses
[distance
])
1383 other
.second
.uses
[i
] -= 1;
1389 if (copy_map
.empty()) {
1390 ctx
->program
->statistics
[statistic_copies
] += ctx
->instructions
.size() - num_instructions_before
;
1394 /* all target regs are needed as operand somewhere which means, all entries are part of a cycle */
1395 unsigned largest
= 0;
1396 for (const std::pair
<const PhysReg
, copy_operation
>& op
: copy_map
)
1397 largest
= MAX2(largest
, op
.second
.bytes
);
1399 while (!copy_map
.empty()) {
1401 /* Perform larger swaps first, because larger swaps swaps can make other
1402 * swaps unnecessary. */
1403 auto it
= copy_map
.begin();
1404 for (auto it2
= copy_map
.begin(); it2
!= copy_map
.end(); ++it2
) {
1405 if (it2
->second
.bytes
> it
->second
.bytes
) {
1407 if (it
->second
.bytes
== largest
)
1412 /* should already be done */
1413 assert(!it
->second
.op
.isConstant());
1415 assert(it
->second
.op
.isFixed());
1416 assert(it
->second
.def
.regClass() == it
->second
.op
.regClass());
1418 if (it
->first
== it
->second
.op
.physReg()) {
1423 if (preserve_scc
&& it
->second
.def
.getTemp().type() == RegType::sgpr
)
1424 assert(!(it
->second
.def
.physReg() == pi
->scratch_sgpr
));
1426 /* to resolve the cycle, we have to swap the src reg with the dst reg */
1427 copy_operation swap
= it
->second
;
1429 /* if this is self-intersecting, we have to split it because
1430 * self-intersecting swaps don't make sense */
1431 PhysReg lower
= swap
.def
.physReg();
1432 PhysReg higher
= swap
.op
.physReg();
1433 if (lower
.reg_b
> higher
.reg_b
)
1434 std::swap(lower
, higher
);
1435 if (higher
.reg_b
- lower
.reg_b
< (int)swap
.bytes
) {
1436 unsigned offset
= higher
.reg_b
- lower
.reg_b
;
1437 RegType type
= swap
.def
.regClass().type();
1439 copy_operation middle
;
1440 lower
.reg_b
+= offset
;
1441 higher
.reg_b
+= offset
;
1442 middle
.bytes
= swap
.bytes
- offset
* 2;
1443 memcpy(middle
.uses
, swap
.uses
+ offset
, middle
.bytes
);
1444 middle
.op
= Operand(lower
, RegClass::get(type
, middle
.bytes
));
1445 middle
.def
= Definition(higher
, RegClass::get(type
, middle
.bytes
));
1446 copy_map
[higher
] = middle
;
1449 lower
.reg_b
+= middle
.bytes
;
1450 higher
.reg_b
+= middle
.bytes
;
1451 end
.bytes
= swap
.bytes
- (offset
+ middle
.bytes
);
1452 memcpy(end
.uses
, swap
.uses
+ offset
+ middle
.bytes
, end
.bytes
);
1453 end
.op
= Operand(lower
, RegClass::get(type
, end
.bytes
));
1454 end
.def
= Definition(higher
, RegClass::get(type
, end
.bytes
));
1455 copy_map
[higher
] = end
;
1457 memset(swap
.uses
+ offset
, 0, swap
.bytes
- offset
);
1458 swap
.bytes
= offset
;
1461 do_swap(ctx
, bld
, swap
, preserve_scc
, pi
);
1463 /* remove from map */
1466 /* change the operand reg of the target's uses and split uses if needed */
1467 target
= copy_map
.begin();
1468 uint32_t bytes_left
= u_bit_consecutive(0, swap
.bytes
);
1469 for (; target
!= copy_map
.end(); ++target
) {
1470 if (target
->second
.op
.physReg() == swap
.def
.physReg() && swap
.bytes
== target
->second
.bytes
) {
1471 target
->second
.op
.setFixed(swap
.op
.physReg());
1475 uint32_t imask
= get_intersection_mask(swap
.def
.physReg().reg_b
, swap
.bytes
,
1476 target
->second
.op
.physReg().reg_b
, target
->second
.bytes
);
1481 assert(target
->second
.bytes
< swap
.bytes
);
1483 int offset
= (int)target
->second
.op
.physReg().reg_b
- (int)swap
.def
.physReg().reg_b
;
1485 /* split and update the middle (the portion that reads the swap's
1486 * definition) to read the swap's operand instead */
1487 int target_op_end
= target
->second
.op
.physReg().reg_b
+ target
->second
.bytes
;
1488 int swap_def_end
= swap
.def
.physReg().reg_b
+ swap
.bytes
;
1489 int before_bytes
= MAX2(-offset
, 0);
1490 int after_bytes
= MAX2(target_op_end
- swap_def_end
, 0);
1491 int middle_bytes
= target
->second
.bytes
- before_bytes
- after_bytes
;
1494 unsigned after_offset
= before_bytes
+ middle_bytes
;
1495 assert(after_offset
> 0);
1496 copy_operation copy
;
1497 copy
.bytes
= after_bytes
;
1498 memcpy(copy
.uses
, target
->second
.uses
+ after_offset
, copy
.bytes
);
1499 RegClass rc
= RegClass::get(target
->second
.op
.regClass().type(), after_bytes
);
1500 copy
.op
= Operand(target
->second
.op
.physReg().advance(after_offset
), rc
);
1501 copy
.def
= Definition(target
->second
.def
.physReg().advance(after_offset
), rc
);
1502 copy_map
[copy
.def
.physReg()] = copy
;
1506 copy_operation copy
;
1507 copy
.bytes
= middle_bytes
;
1508 memcpy(copy
.uses
, target
->second
.uses
+ before_bytes
, copy
.bytes
);
1509 RegClass rc
= RegClass::get(target
->second
.op
.regClass().type(), middle_bytes
);
1510 copy
.op
= Operand(swap
.op
.physReg().advance(MAX2(offset
, 0)), rc
);
1511 copy
.def
= Definition(target
->second
.def
.physReg().advance(before_bytes
), rc
);
1512 copy_map
[copy
.def
.physReg()] = copy
;
1516 copy_operation copy
;
1517 target
->second
.bytes
= before_bytes
;
1518 RegClass rc
= RegClass::get(target
->second
.op
.regClass().type(), before_bytes
);
1519 target
->second
.op
= Operand(target
->second
.op
.physReg(), rc
);
1520 target
->second
.def
= Definition(target
->second
.def
.physReg(), rc
);
1521 memset(target
->second
.uses
+ target
->second
.bytes
, 0, 8 - target
->second
.bytes
);
1524 /* break early since we know each byte of the swap's definition is used
1526 bytes_left
&= ~imask
;
1531 ctx
->program
->statistics
[statistic_copies
] += ctx
->instructions
.size() - num_instructions_before
;
1534 void lower_to_hw_instr(Program
* program
)
1536 Block
*discard_block
= NULL
;
1538 for (size_t i
= 0; i
< program
->blocks
.size(); i
++)
1540 Block
*block
= &program
->blocks
[i
];
1542 ctx
.program
= program
;
1543 Builder
bld(program
, &ctx
.instructions
);
1545 bool set_mode
= i
== 0 && block
->fp_mode
.val
!= program
->config
->float_mode
;
1546 for (unsigned pred
: block
->linear_preds
) {
1547 if (program
->blocks
[pred
].fp_mode
.val
!= block
->fp_mode
.val
) {
1553 /* only allow changing modes at top-level blocks so this doesn't break
1554 * the "jump over empty blocks" optimization */
1555 assert(block
->kind
& block_kind_top_level
);
1556 uint32_t mode
= block
->fp_mode
.val
;
1557 /* "((size - 1) << 11) | register" (MODE is encoded as register 1) */
1558 bld
.sopk(aco_opcode::s_setreg_imm32_b32
, Operand(mode
), (7 << 11) | 1);
1561 for (size_t j
= 0; j
< block
->instructions
.size(); j
++) {
1562 aco_ptr
<Instruction
>& instr
= block
->instructions
[j
];
1563 aco_ptr
<Instruction
> mov
;
1564 if (instr
->format
== Format::PSEUDO
) {
1565 Pseudo_instruction
*pi
= (Pseudo_instruction
*)instr
.get();
1567 switch (instr
->opcode
)
1569 case aco_opcode::p_extract_vector
:
1571 PhysReg reg
= instr
->operands
[0].physReg();
1572 Definition
& def
= instr
->definitions
[0];
1573 reg
.reg_b
+= instr
->operands
[1].constantValue() * def
.bytes();
1575 if (reg
== def
.physReg())
1578 RegClass op_rc
= def
.regClass().is_subdword() ? def
.regClass() :
1579 RegClass(instr
->operands
[0].getTemp().type(), def
.size());
1580 std::map
<PhysReg
, copy_operation
> copy_operations
;
1581 copy_operations
[def
.physReg()] = {Operand(reg
, op_rc
), def
, def
.bytes()};
1582 handle_operands(copy_operations
, &ctx
, program
->chip_class
, pi
);
1585 case aco_opcode::p_create_vector
:
1587 std::map
<PhysReg
, copy_operation
> copy_operations
;
1588 PhysReg reg
= instr
->definitions
[0].physReg();
1590 for (const Operand
& op
: instr
->operands
) {
1591 if (op
.isConstant()) {
1592 const Definition def
= Definition(reg
, RegClass(instr
->definitions
[0].getTemp().type(), op
.size()));
1593 copy_operations
[reg
] = {op
, def
, op
.bytes()};
1594 reg
.reg_b
+= op
.bytes();
1597 if (op
.isUndefined()) {
1598 // TODO: coalesce subdword copies if dst byte is 0
1599 reg
.reg_b
+= op
.bytes();
1603 RegClass rc_def
= op
.regClass().is_subdword() ? op
.regClass() :
1604 RegClass(instr
->definitions
[0].getTemp().type(), op
.size());
1605 const Definition def
= Definition(reg
, rc_def
);
1606 copy_operations
[def
.physReg()] = {op
, def
, op
.bytes()};
1607 reg
.reg_b
+= op
.bytes();
1609 handle_operands(copy_operations
, &ctx
, program
->chip_class
, pi
);
1612 case aco_opcode::p_split_vector
:
1614 std::map
<PhysReg
, copy_operation
> copy_operations
;
1615 PhysReg reg
= instr
->operands
[0].physReg();
1617 for (const Definition
& def
: instr
->definitions
) {
1618 RegClass rc_op
= def
.regClass().is_subdword() ? def
.regClass() :
1619 RegClass(instr
->operands
[0].getTemp().type(), def
.size());
1620 const Operand op
= Operand(reg
, rc_op
);
1621 copy_operations
[def
.physReg()] = {op
, def
, def
.bytes()};
1622 reg
.reg_b
+= def
.bytes();
1624 handle_operands(copy_operations
, &ctx
, program
->chip_class
, pi
);
1627 case aco_opcode::p_parallelcopy
:
1628 case aco_opcode::p_wqm
:
1630 std::map
<PhysReg
, copy_operation
> copy_operations
;
1631 for (unsigned i
= 0; i
< instr
->operands
.size(); i
++) {
1632 assert(instr
->definitions
[i
].bytes() == instr
->operands
[i
].bytes());
1633 copy_operations
[instr
->definitions
[i
].physReg()] = {instr
->operands
[i
], instr
->definitions
[i
], instr
->operands
[i
].bytes()};
1635 handle_operands(copy_operations
, &ctx
, program
->chip_class
, pi
);
1638 case aco_opcode::p_exit_early_if
:
1640 /* don't bother with an early exit near the end of the program */
1641 if ((block
->instructions
.size() - 1 - j
) <= 4 &&
1642 block
->instructions
.back()->opcode
== aco_opcode::s_endpgm
) {
1643 unsigned null_exp_dest
= (ctx
.program
->stage
& hw_fs
) ? 9 /* NULL */ : V_008DFC_SQ_EXP_POS
;
1644 bool ignore_early_exit
= true;
1646 for (unsigned k
= j
+ 1; k
< block
->instructions
.size(); ++k
) {
1647 const aco_ptr
<Instruction
> &instr
= block
->instructions
[k
];
1648 if (instr
->opcode
== aco_opcode::s_endpgm
||
1649 instr
->opcode
== aco_opcode::p_logical_end
)
1651 else if (instr
->opcode
== aco_opcode::exp
&&
1652 static_cast<Export_instruction
*>(instr
.get())->dest
== null_exp_dest
)
1654 else if (instr
->opcode
== aco_opcode::p_parallelcopy
&&
1655 instr
->definitions
[0].isFixed() &&
1656 instr
->definitions
[0].physReg() == exec
)
1659 ignore_early_exit
= false;
1662 if (ignore_early_exit
)
1666 if (!discard_block
) {
1667 discard_block
= program
->create_and_insert_block();
1668 block
= &program
->blocks
[i
];
1670 bld
.reset(discard_block
);
1671 bld
.exp(aco_opcode::exp
, Operand(v1
), Operand(v1
), Operand(v1
), Operand(v1
),
1672 0, V_008DFC_SQ_EXP_NULL
, false, true, true);
1673 if (program
->wb_smem_l1_on_end
)
1674 bld
.smem(aco_opcode::s_dcache_wb
);
1675 bld
.sopp(aco_opcode::s_endpgm
);
1677 bld
.reset(&ctx
.instructions
);
1680 //TODO: exec can be zero here with block_kind_discard
1682 assert(instr
->operands
[0].physReg() == scc
);
1683 bld
.sopp(aco_opcode::s_cbranch_scc0
, instr
->operands
[0], discard_block
->index
);
1685 discard_block
->linear_preds
.push_back(block
->index
);
1686 block
->linear_succs
.push_back(discard_block
->index
);
1689 case aco_opcode::p_spill
:
1691 assert(instr
->operands
[0].regClass() == v1
.as_linear());
1692 for (unsigned i
= 0; i
< instr
->operands
[2].size(); i
++)
1693 bld
.writelane(bld
.def(v1
, instr
->operands
[0].physReg()),
1694 Operand(PhysReg
{instr
->operands
[2].physReg() + i
}, s1
),
1695 Operand(instr
->operands
[1].constantValue() + i
),
1696 instr
->operands
[0]);
1699 case aco_opcode::p_reload
:
1701 assert(instr
->operands
[0].regClass() == v1
.as_linear());
1702 for (unsigned i
= 0; i
< instr
->definitions
[0].size(); i
++)
1703 bld
.readlane(bld
.def(s1
, PhysReg
{instr
->definitions
[0].physReg() + i
}),
1705 Operand(instr
->operands
[1].constantValue() + i
));
1708 case aco_opcode::p_as_uniform
:
1710 if (instr
->operands
[0].isConstant() || instr
->operands
[0].regClass().type() == RegType::sgpr
) {
1711 std::map
<PhysReg
, copy_operation
> copy_operations
;
1712 copy_operations
[instr
->definitions
[0].physReg()] = {instr
->operands
[0], instr
->definitions
[0], instr
->definitions
[0].bytes()};
1713 handle_operands(copy_operations
, &ctx
, program
->chip_class
, pi
);
1715 assert(instr
->operands
[0].regClass().type() == RegType::vgpr
);
1716 assert(instr
->definitions
[0].regClass().type() == RegType::sgpr
);
1717 assert(instr
->operands
[0].size() == instr
->definitions
[0].size());
1718 for (unsigned i
= 0; i
< instr
->definitions
[0].size(); i
++) {
1719 bld
.vop1(aco_opcode::v_readfirstlane_b32
,
1720 bld
.def(s1
, PhysReg
{instr
->definitions
[0].physReg() + i
}),
1721 Operand(PhysReg
{instr
->operands
[0].physReg() + i
}, v1
));
1726 case aco_opcode::p_bpermute
:
1728 if (ctx
.program
->chip_class
<= GFX7
)
1729 emit_gfx6_bpermute(program
, instr
, bld
);
1730 else if (ctx
.program
->chip_class
== GFX10
&& ctx
.program
->wave_size
== 64)
1731 emit_gfx10_wave64_bpermute(program
, instr
, bld
);
1733 unreachable("Current hardware supports ds_bpermute, don't emit p_bpermute.");
1738 } else if (instr
->format
== Format::PSEUDO_BRANCH
) {
1739 Pseudo_branch_instruction
* branch
= static_cast<Pseudo_branch_instruction
*>(instr
.get());
1740 /* check if all blocks from current to target are empty */
1741 bool can_remove
= block
->index
< branch
->target
[0];
1742 for (unsigned i
= block
->index
+ 1; can_remove
&& i
< branch
->target
[0]; i
++) {
1743 if (program
->blocks
[i
].instructions
.size())
1749 switch (instr
->opcode
) {
1750 case aco_opcode::p_branch
:
1751 assert(block
->linear_succs
[0] == branch
->target
[0]);
1752 bld
.sopp(aco_opcode::s_branch
, branch
->target
[0]);
1754 case aco_opcode::p_cbranch_nz
:
1755 assert(block
->linear_succs
[1] == branch
->target
[0]);
1756 if (branch
->operands
[0].physReg() == exec
)
1757 bld
.sopp(aco_opcode::s_cbranch_execnz
, branch
->target
[0]);
1758 else if (branch
->operands
[0].physReg() == vcc
)
1759 bld
.sopp(aco_opcode::s_cbranch_vccnz
, branch
->target
[0]);
1761 assert(branch
->operands
[0].physReg() == scc
);
1762 bld
.sopp(aco_opcode::s_cbranch_scc1
, branch
->target
[0]);
1765 case aco_opcode::p_cbranch_z
:
1766 assert(block
->linear_succs
[1] == branch
->target
[0]);
1767 if (branch
->operands
[0].physReg() == exec
)
1768 bld
.sopp(aco_opcode::s_cbranch_execz
, branch
->target
[0]);
1769 else if (branch
->operands
[0].physReg() == vcc
)
1770 bld
.sopp(aco_opcode::s_cbranch_vccz
, branch
->target
[0]);
1772 assert(branch
->operands
[0].physReg() == scc
);
1773 bld
.sopp(aco_opcode::s_cbranch_scc0
, branch
->target
[0]);
1777 unreachable("Unknown Pseudo branch instruction!");
1780 } else if (instr
->format
== Format::PSEUDO_REDUCTION
) {
1781 Pseudo_reduction_instruction
* reduce
= static_cast<Pseudo_reduction_instruction
*>(instr
.get());
1782 emit_reduction(&ctx
, reduce
->opcode
, reduce
->reduce_op
, reduce
->cluster_size
,
1783 reduce
->operands
[1].physReg(), // tmp
1784 reduce
->definitions
[1].physReg(), // stmp
1785 reduce
->operands
[2].physReg(), // vtmp
1786 reduce
->definitions
[2].physReg(), // sitmp
1787 reduce
->operands
[0], reduce
->definitions
[0]);
1789 ctx
.instructions
.emplace_back(std::move(instr
));
1793 block
->instructions
.swap(ctx
.instructions
);