2 * Copyright © 2018 Valve Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Daniel Schürmann (daniel.schuermann@campus.tu-berlin.de)
31 #include "aco_builder.h"
32 #include "util/u_math.h"
34 #include "vulkan/radv_shader.h"
39 struct lower_context
{
41 std::vector
<aco_ptr
<Instruction
>> instructions
;
44 aco_opcode
get_reduce_opcode(chip_class chip
, ReduceOp op
) {
46 case iadd32
: return chip
>= GFX9
? aco_opcode::v_add_u32
: aco_opcode::v_add_co_u32
;
47 case imul32
: return aco_opcode::v_mul_lo_u32
;
48 case fadd32
: return aco_opcode::v_add_f32
;
49 case fmul32
: return aco_opcode::v_mul_f32
;
50 case imax32
: return aco_opcode::v_max_i32
;
51 case imin32
: return aco_opcode::v_min_i32
;
52 case umin32
: return aco_opcode::v_min_u32
;
53 case umax32
: return aco_opcode::v_max_u32
;
54 case fmin32
: return aco_opcode::v_min_f32
;
55 case fmax32
: return aco_opcode::v_max_f32
;
56 case iand32
: return aco_opcode::v_and_b32
;
57 case ixor32
: return aco_opcode::v_xor_b32
;
58 case ior32
: return aco_opcode::v_or_b32
;
59 case iadd64
: return aco_opcode::num_opcodes
;
60 case imul64
: return aco_opcode::num_opcodes
;
61 case fadd64
: return aco_opcode::v_add_f64
;
62 case fmul64
: return aco_opcode::v_mul_f64
;
63 case imin64
: return aco_opcode::num_opcodes
;
64 case imax64
: return aco_opcode::num_opcodes
;
65 case umin64
: return aco_opcode::num_opcodes
;
66 case umax64
: return aco_opcode::num_opcodes
;
67 case fmin64
: return aco_opcode::v_min_f64
;
68 case fmax64
: return aco_opcode::v_max_f64
;
69 case iand64
: return aco_opcode::num_opcodes
;
70 case ior64
: return aco_opcode::num_opcodes
;
71 case ixor64
: return aco_opcode::num_opcodes
;
72 default: return aco_opcode::num_opcodes
;
76 void emit_vadd32(Builder
& bld
, Definition def
, Operand src0
, Operand src1
)
78 Instruction
*instr
= bld
.vadd32(def
, src0
, src1
, false, Operand(s2
), true);
79 if (instr
->definitions
.size() >= 2) {
80 assert(instr
->definitions
[1].regClass() == bld
.lm
);
81 instr
->definitions
[1].setFixed(vcc
);
85 void emit_int64_dpp_op(lower_context
*ctx
, PhysReg dst_reg
, PhysReg src0_reg
, PhysReg src1_reg
,
86 PhysReg vtmp_reg
, ReduceOp op
,
87 unsigned dpp_ctrl
, unsigned row_mask
, unsigned bank_mask
, bool bound_ctrl
,
88 Operand
*identity
=NULL
)
90 Builder
bld(ctx
->program
, &ctx
->instructions
);
91 Definition dst
[] = {Definition(dst_reg
, v1
), Definition(PhysReg
{dst_reg
+1}, v1
)};
92 Definition vtmp_def
[] = {Definition(vtmp_reg
, v1
), Definition(PhysReg
{vtmp_reg
+1}, v1
)};
93 Operand src0
[] = {Operand(src0_reg
, v1
), Operand(PhysReg
{src0_reg
+1}, v1
)};
94 Operand src1
[] = {Operand(src1_reg
, v1
), Operand(PhysReg
{src1_reg
+1}, v1
)};
95 Operand src1_64
= Operand(src1_reg
, v2
);
96 Operand vtmp_op
[] = {Operand(vtmp_reg
, v1
), Operand(PhysReg
{vtmp_reg
+1}, v1
)};
97 Operand vtmp_op64
= Operand(vtmp_reg
, v2
);
99 if (ctx
->program
->chip_class
>= GFX10
) {
101 bld
.vop1(aco_opcode::v_mov_b32
, vtmp_def
[0], identity
[0]);
102 bld
.vop1_dpp(aco_opcode::v_mov_b32
, vtmp_def
[0], src0
[0],
103 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
104 bld
.vop3(aco_opcode::v_add_co_u32_e64
, dst
[0], bld
.def(bld
.lm
, vcc
), vtmp_op
[0], src1
[0]);
106 bld
.vop2_dpp(aco_opcode::v_add_co_u32
, dst
[0], bld
.def(bld
.lm
, vcc
), src0
[0], src1
[0],
107 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
109 bld
.vop2_dpp(aco_opcode::v_addc_co_u32
, dst
[1], bld
.def(bld
.lm
, vcc
), src0
[1], src1
[1], Operand(vcc
, bld
.lm
),
110 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
111 } else if (op
== iand64
) {
112 bld
.vop2_dpp(aco_opcode::v_and_b32
, dst
[0], src0
[0], src1
[0],
113 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
114 bld
.vop2_dpp(aco_opcode::v_and_b32
, dst
[1], src0
[1], src1
[1],
115 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
116 } else if (op
== ior64
) {
117 bld
.vop2_dpp(aco_opcode::v_or_b32
, dst
[0], src0
[0], src1
[0],
118 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
119 bld
.vop2_dpp(aco_opcode::v_or_b32
, dst
[1], src0
[1], src1
[1],
120 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
121 } else if (op
== ixor64
) {
122 bld
.vop2_dpp(aco_opcode::v_xor_b32
, dst
[0], src0
[0], src1
[0],
123 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
124 bld
.vop2_dpp(aco_opcode::v_xor_b32
, dst
[1], src0
[1], src1
[1],
125 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
126 } else if (op
== umin64
|| op
== umax64
|| op
== imin64
|| op
== imax64
) {
127 aco_opcode cmp
= aco_opcode::num_opcodes
;
130 cmp
= aco_opcode::v_cmp_gt_u64
;
133 cmp
= aco_opcode::v_cmp_lt_u64
;
136 cmp
= aco_opcode::v_cmp_gt_i64
;
139 cmp
= aco_opcode::v_cmp_lt_i64
;
146 bld
.vop1(aco_opcode::v_mov_b32
, vtmp_def
[0], identity
[0]);
147 bld
.vop1(aco_opcode::v_mov_b32
, vtmp_def
[1], identity
[1]);
149 bld
.vop1_dpp(aco_opcode::v_mov_b32
, vtmp_def
[0], src0
[0],
150 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
151 bld
.vop1_dpp(aco_opcode::v_mov_b32
, vtmp_def
[1], src0
[1],
152 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
154 bld
.vopc(cmp
, bld
.def(bld
.lm
, vcc
), vtmp_op64
, src1_64
);
155 bld
.vop2(aco_opcode::v_cndmask_b32
, dst
[0], vtmp_op
[0], src1
[0], Operand(vcc
, bld
.lm
));
156 bld
.vop2(aco_opcode::v_cndmask_b32
, dst
[1], vtmp_op
[1], src1
[1], Operand(vcc
, bld
.lm
));
157 } else if (op
== imul64
) {
159 * t1 = umul_lo(t4, y_lo)
161 * t0 = umul_lo(t3, y_hi)
163 * t5 = umul_hi(t3, y_lo)
164 * res_hi = iadd(t2, t5)
165 * res_lo = umul_lo(t3, y_lo)
166 * Requires that res_hi != src0[0] and res_hi != src1[0]
167 * and that vtmp[0] != res_hi.
170 bld
.vop1(aco_opcode::v_mov_b32
, vtmp_def
[0], identity
[1]);
171 bld
.vop1_dpp(aco_opcode::v_mov_b32
, vtmp_def
[0], src0
[1],
172 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
173 bld
.vop3(aco_opcode::v_mul_lo_u32
, vtmp_def
[1], vtmp_op
[0], src1
[0]);
175 bld
.vop1(aco_opcode::v_mov_b32
, vtmp_def
[0], identity
[0]);
176 bld
.vop1_dpp(aco_opcode::v_mov_b32
, vtmp_def
[0], src0
[0],
177 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
178 bld
.vop3(aco_opcode::v_mul_lo_u32
, vtmp_def
[0], vtmp_op
[0], src1
[1]);
179 emit_vadd32(bld
, vtmp_def
[1], vtmp_op
[0], vtmp_op
[1]);
181 bld
.vop1(aco_opcode::v_mov_b32
, vtmp_def
[0], identity
[0]);
182 bld
.vop1_dpp(aco_opcode::v_mov_b32
, vtmp_def
[0], src0
[0],
183 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
184 bld
.vop3(aco_opcode::v_mul_hi_u32
, vtmp_def
[0], vtmp_op
[0], src1
[0]);
185 emit_vadd32(bld
, dst
[1], vtmp_op
[1], vtmp_op
[0]);
187 bld
.vop1(aco_opcode::v_mov_b32
, vtmp_def
[0], identity
[0]);
188 bld
.vop1_dpp(aco_opcode::v_mov_b32
, vtmp_def
[0], src0
[0],
189 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
190 bld
.vop3(aco_opcode::v_mul_lo_u32
, dst
[0], vtmp_op
[0], src1
[0]);
194 void emit_int64_op(lower_context
*ctx
, PhysReg dst_reg
, PhysReg src0_reg
, PhysReg src1_reg
, PhysReg vtmp
, ReduceOp op
)
196 Builder
bld(ctx
->program
, &ctx
->instructions
);
197 Definition dst
[] = {Definition(dst_reg
, v1
), Definition(PhysReg
{dst_reg
+1}, v1
)};
198 RegClass src0_rc
= src0_reg
.reg() >= 256 ? v1
: s1
;
199 Operand src0
[] = {Operand(src0_reg
, src0_rc
), Operand(PhysReg
{src0_reg
+1}, src0_rc
)};
200 Operand src1
[] = {Operand(src1_reg
, v1
), Operand(PhysReg
{src1_reg
+1}, v1
)};
201 Operand src0_64
= Operand(src0_reg
, src0_reg
.reg() >= 256 ? v2
: s2
);
202 Operand src1_64
= Operand(src1_reg
, v2
);
205 (op
== imul64
|| op
== umin64
|| op
== umax64
|| op
== imin64
|| op
== imax64
)) {
206 assert(vtmp
.reg() != 0);
207 bld
.vop1(aco_opcode::v_mov_b32
, Definition(vtmp
, v1
), src0
[0]);
208 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{vtmp
+1}, v1
), src0
[1]);
210 src0
[0] = Operand(vtmp
, v1
);
211 src0
[1] = Operand(PhysReg
{vtmp
+1}, v1
);
212 src0_64
= Operand(vtmp
, v2
);
213 } else if (src0_rc
== s1
&& op
== iadd64
) {
214 assert(vtmp
.reg() != 0);
215 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{vtmp
+1}, v1
), src0
[1]);
216 src0
[1] = Operand(PhysReg
{vtmp
+1}, v1
);
220 if (ctx
->program
->chip_class
>= GFX10
) {
221 bld
.vop3(aco_opcode::v_add_co_u32_e64
, dst
[0], bld
.def(bld
.lm
, vcc
), src0
[0], src1
[0]);
223 bld
.vop2(aco_opcode::v_add_co_u32
, dst
[0], bld
.def(bld
.lm
, vcc
), src0
[0], src1
[0]);
225 bld
.vop2(aco_opcode::v_addc_co_u32
, dst
[1], bld
.def(bld
.lm
, vcc
), src0
[1], src1
[1], Operand(vcc
, bld
.lm
));
226 } else if (op
== iand64
) {
227 bld
.vop2(aco_opcode::v_and_b32
, dst
[0], src0
[0], src1
[0]);
228 bld
.vop2(aco_opcode::v_and_b32
, dst
[1], src0
[1], src1
[1]);
229 } else if (op
== ior64
) {
230 bld
.vop2(aco_opcode::v_or_b32
, dst
[0], src0
[0], src1
[0]);
231 bld
.vop2(aco_opcode::v_or_b32
, dst
[1], src0
[1], src1
[1]);
232 } else if (op
== ixor64
) {
233 bld
.vop2(aco_opcode::v_xor_b32
, dst
[0], src0
[0], src1
[0]);
234 bld
.vop2(aco_opcode::v_xor_b32
, dst
[1], src0
[1], src1
[1]);
235 } else if (op
== umin64
|| op
== umax64
|| op
== imin64
|| op
== imax64
) {
236 aco_opcode cmp
= aco_opcode::num_opcodes
;
239 cmp
= aco_opcode::v_cmp_gt_u64
;
242 cmp
= aco_opcode::v_cmp_lt_u64
;
245 cmp
= aco_opcode::v_cmp_gt_i64
;
248 cmp
= aco_opcode::v_cmp_lt_i64
;
254 bld
.vopc(cmp
, bld
.def(bld
.lm
, vcc
), src0_64
, src1_64
);
255 bld
.vop2(aco_opcode::v_cndmask_b32
, dst
[0], src0
[0], src1
[0], Operand(vcc
, bld
.lm
));
256 bld
.vop2(aco_opcode::v_cndmask_b32
, dst
[1], src0
[1], src1
[1], Operand(vcc
, bld
.lm
));
257 } else if (op
== imul64
) {
258 if (src1_reg
== dst_reg
) {
259 /* it's fine if src0==dst but not if src1==dst */
260 std::swap(src0_reg
, src1_reg
);
261 std::swap(src0
[0], src1
[0]);
262 std::swap(src0
[1], src1
[1]);
263 std::swap(src0_64
, src1_64
);
265 assert(!(src0_reg
== src1_reg
));
266 /* t1 = umul_lo(x_hi, y_lo)
267 * t0 = umul_lo(x_lo, y_hi)
269 * t5 = umul_hi(x_lo, y_lo)
270 * res_hi = iadd(t2, t5)
271 * res_lo = umul_lo(x_lo, y_lo)
272 * assumes that it's ok to modify x_hi/y_hi, since we might not have vtmp
274 Definition
tmp0_def(PhysReg
{src0_reg
+1}, v1
);
275 Definition
tmp1_def(PhysReg
{src1_reg
+1}, v1
);
276 Operand tmp0_op
= src0
[1];
277 Operand tmp1_op
= src1
[1];
278 bld
.vop3(aco_opcode::v_mul_lo_u32
, tmp0_def
, src0
[1], src1
[0]);
279 bld
.vop3(aco_opcode::v_mul_lo_u32
, tmp1_def
, src0
[0], src1
[1]);
280 emit_vadd32(bld
, tmp0_def
, tmp1_op
, tmp0_op
);
281 bld
.vop3(aco_opcode::v_mul_hi_u32
, tmp1_def
, src0
[0], src1
[0]);
282 emit_vadd32(bld
, dst
[1], tmp0_op
, tmp1_op
);
283 bld
.vop3(aco_opcode::v_mul_lo_u32
, dst
[0], src0
[0], src1
[0]);
287 void emit_dpp_op(lower_context
*ctx
, PhysReg dst_reg
, PhysReg src0_reg
, PhysReg src1_reg
,
288 PhysReg vtmp
, ReduceOp op
, unsigned size
,
289 unsigned dpp_ctrl
, unsigned row_mask
, unsigned bank_mask
, bool bound_ctrl
,
290 Operand
*identity
=NULL
) /* for VOP3 with sparse writes */
292 Builder
bld(ctx
->program
, &ctx
->instructions
);
293 RegClass rc
= RegClass(RegType::vgpr
, size
);
294 Definition
dst(dst_reg
, rc
);
295 Operand
src0(src0_reg
, rc
);
296 Operand
src1(src1_reg
, rc
);
298 aco_opcode opcode
= get_reduce_opcode(ctx
->program
->chip_class
, op
);
299 bool vop3
= op
== imul32
|| size
== 2;
302 if (opcode
== aco_opcode::v_add_co_u32
)
303 bld
.vop2_dpp(opcode
, dst
, bld
.def(bld
.lm
, vcc
), src0
, src1
, dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
305 bld
.vop2_dpp(opcode
, dst
, src0
, src1
, dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
309 if (opcode
== aco_opcode::num_opcodes
) {
310 emit_int64_dpp_op(ctx
, dst_reg
,src0_reg
, src1_reg
, vtmp
, op
,
311 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
, identity
);
316 bld
.vop1(aco_opcode::v_mov_b32
, Definition(vtmp
, v1
), identity
[0]);
317 if (identity
&& size
>= 2)
318 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{vtmp
+1}, v1
), identity
[1]);
320 for (unsigned i
= 0; i
< size
; i
++)
321 bld
.vop1_dpp(aco_opcode::v_mov_b32
, Definition(PhysReg
{vtmp
+i
}, v1
), Operand(PhysReg
{src0_reg
+i
}, v1
),
322 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
324 bld
.vop3(opcode
, dst
, Operand(vtmp
, rc
), src1
);
327 void emit_op(lower_context
*ctx
, PhysReg dst_reg
, PhysReg src0_reg
, PhysReg src1_reg
,
328 PhysReg vtmp
, ReduceOp op
, unsigned size
)
330 Builder
bld(ctx
->program
, &ctx
->instructions
);
331 RegClass rc
= RegClass(RegType::vgpr
, size
);
332 Definition
dst(dst_reg
, rc
);
333 Operand
src0(src0_reg
, RegClass(src0_reg
.reg() >= 256 ? RegType::vgpr
: RegType::sgpr
, size
));
334 Operand
src1(src1_reg
, rc
);
336 aco_opcode opcode
= get_reduce_opcode(ctx
->program
->chip_class
, op
);
337 bool vop3
= op
== imul32
|| size
== 2;
339 if (opcode
== aco_opcode::num_opcodes
) {
340 emit_int64_op(ctx
, dst_reg
, src0_reg
, src1_reg
, vtmp
, op
);
345 bld
.vop3(opcode
, dst
, src0
, src1
);
346 } else if (opcode
== aco_opcode::v_add_co_u32
) {
347 bld
.vop2(opcode
, dst
, bld
.def(bld
.lm
, vcc
), src0
, src1
);
349 bld
.vop2(opcode
, dst
, src0
, src1
);
353 void emit_dpp_mov(lower_context
*ctx
, PhysReg dst
, PhysReg src0
, unsigned size
,
354 unsigned dpp_ctrl
, unsigned row_mask
, unsigned bank_mask
, bool bound_ctrl
)
356 Builder
bld(ctx
->program
, &ctx
->instructions
);
357 for (unsigned i
= 0; i
< size
; i
++) {
358 bld
.vop1_dpp(aco_opcode::v_mov_b32
, Definition(PhysReg
{dst
+i
}, v1
), Operand(PhysReg
{src0
+i
}, v1
),
359 dpp_ctrl
, row_mask
, bank_mask
, bound_ctrl
);
363 uint32_t get_reduction_identity(ReduceOp op
, unsigned idx
)
381 return 0x3f800000u
; /* 1.0 */
383 return idx
? 0x3ff00000u
: 0u; /* 1.0 */
387 return idx
? 0x7fffffffu
: 0xffffffffu
;
391 return idx
? 0x80000000u
: 0;
398 return 0x7f800000u
; /* infinity */
400 return idx
? 0x7ff00000u
: 0u; /* infinity */
402 return 0xff800000u
; /* negative infinity */
404 return idx
? 0xfff00000u
: 0u; /* negative infinity */
406 unreachable("Invalid reduction operation");
412 void emit_ds_swizzle(Builder bld
, PhysReg dst
, PhysReg src
, unsigned size
, unsigned ds_pattern
)
414 for (unsigned i
= 0; i
< size
; i
++) {
415 bld
.ds(aco_opcode::ds_swizzle_b32
, Definition(PhysReg
{dst
+i
}, v1
),
416 Operand(PhysReg
{src
+i
}, v1
), ds_pattern
);
420 void emit_reduction(lower_context
*ctx
, aco_opcode op
, ReduceOp reduce_op
, unsigned cluster_size
, PhysReg tmp
,
421 PhysReg stmp
, PhysReg vtmp
, PhysReg sitmp
, Operand src
, Definition dst
)
423 assert(cluster_size
== ctx
->program
->wave_size
|| op
== aco_opcode::p_reduce
);
424 assert(cluster_size
<= ctx
->program
->wave_size
);
426 Builder
bld(ctx
->program
, &ctx
->instructions
);
429 identity
[0] = Operand(get_reduction_identity(reduce_op
, 0));
430 identity
[1] = Operand(get_reduction_identity(reduce_op
, 1));
431 Operand vcndmask_identity
[2] = {identity
[0], identity
[1]};
433 /* First, copy the source to tmp and set inactive lanes to the identity */
434 bld
.sop1(Builder::s_or_saveexec
, Definition(stmp
, bld
.lm
), Definition(scc
, s1
), Definition(exec
, bld
.lm
), Operand(UINT64_MAX
), Operand(exec
, bld
.lm
));
436 for (unsigned i
= 0; i
< src
.size(); i
++) {
437 /* p_exclusive_scan needs it to be a sgpr or inline constant for the v_writelane_b32
438 * except on GFX10, where v_writelane_b32 can take a literal. */
439 if (identity
[i
].isLiteral() && op
== aco_opcode::p_exclusive_scan
&& ctx
->program
->chip_class
< GFX10
) {
440 bld
.sop1(aco_opcode::s_mov_b32
, Definition(PhysReg
{sitmp
+i
}, s1
), identity
[i
]);
441 identity
[i
] = Operand(PhysReg
{sitmp
+i
}, s1
);
443 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{tmp
+i
}, v1
), identity
[i
]);
444 vcndmask_identity
[i
] = Operand(PhysReg
{tmp
+i
}, v1
);
445 } else if (identity
[i
].isLiteral()) {
446 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{tmp
+i
}, v1
), identity
[i
]);
447 vcndmask_identity
[i
] = Operand(PhysReg
{tmp
+i
}, v1
);
451 for (unsigned i
= 0; i
< src
.size(); i
++) {
452 bld
.vop2_e64(aco_opcode::v_cndmask_b32
, Definition(PhysReg
{tmp
+ i
}, v1
),
453 vcndmask_identity
[i
], Operand(PhysReg
{src
.physReg() + i
}, v1
),
454 Operand(stmp
, bld
.lm
));
457 bool reduction_needs_last_op
= false;
459 case aco_opcode::p_reduce
:
460 if (cluster_size
== 1) break;
462 if (ctx
->program
->chip_class
<= GFX7
) {
463 reduction_needs_last_op
= true;
464 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), (1 << 15) | dpp_quad_perm(1, 0, 3, 2));
465 if (cluster_size
== 2) break;
466 emit_op(ctx
, tmp
, vtmp
, tmp
, PhysReg
{0}, reduce_op
, src
.size());
467 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), (1 << 15) | dpp_quad_perm(2, 3, 0, 1));
468 if (cluster_size
== 4) break;
469 emit_op(ctx
, tmp
, vtmp
, tmp
, PhysReg
{0}, reduce_op
, src
.size());
470 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1f, 0, 0x04));
471 if (cluster_size
== 8) break;
472 emit_op(ctx
, tmp
, vtmp
, tmp
, PhysReg
{0}, reduce_op
, src
.size());
473 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1f, 0, 0x08));
474 if (cluster_size
== 16) break;
475 emit_op(ctx
, tmp
, vtmp
, tmp
, PhysReg
{0}, reduce_op
, src
.size());
476 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1f, 0, 0x10));
477 if (cluster_size
== 32) break;
478 emit_op(ctx
, tmp
, vtmp
, tmp
, PhysReg
{0}, reduce_op
, src
.size());
479 for (unsigned i
= 0; i
< src
.size(); i
++)
480 bld
.readlane(Definition(PhysReg
{dst
.physReg() + i
}, s1
), Operand(PhysReg
{tmp
+ i
}, v1
), Operand(0u));
481 // TODO: it would be more effective to do the last reduction step on SALU
482 emit_op(ctx
, tmp
, dst
.physReg(), tmp
, vtmp
, reduce_op
, src
.size());
483 reduction_needs_last_op
= false;
487 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(), dpp_quad_perm(1, 0, 3, 2), 0xf, 0xf, false);
488 if (cluster_size
== 2) break;
489 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(), dpp_quad_perm(2, 3, 0, 1), 0xf, 0xf, false);
490 if (cluster_size
== 4) break;
491 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(), dpp_row_half_mirror
, 0xf, 0xf, false);
492 if (cluster_size
== 8) break;
493 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(), dpp_row_mirror
, 0xf, 0xf, false);
494 if (cluster_size
== 16) break;
496 if (ctx
->program
->chip_class
>= GFX10
) {
497 /* GFX10+ doesn't support row_bcast15 and row_bcast31 */
498 for (unsigned i
= 0; i
< src
.size(); i
++)
499 bld
.vop3(aco_opcode::v_permlanex16_b32
, Definition(PhysReg
{vtmp
+i
}, v1
), Operand(PhysReg
{tmp
+i
}, v1
), Operand(0u), Operand(0u));
501 if (cluster_size
== 32) {
502 reduction_needs_last_op
= true;
506 emit_op(ctx
, tmp
, tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
507 for (unsigned i
= 0; i
< src
.size(); i
++)
508 bld
.readlane(Definition(PhysReg
{dst
.physReg() + i
}, s1
), Operand(PhysReg
{tmp
+i
}, v1
), Operand(0u));
509 // TODO: it would be more effective to do the last reduction step on SALU
510 emit_op(ctx
, tmp
, dst
.physReg(), tmp
, vtmp
, reduce_op
, src
.size());
514 if (cluster_size
== 32) {
515 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1f, 0, 0x10));
516 reduction_needs_last_op
= true;
519 assert(cluster_size
== 64);
520 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(), dpp_row_bcast15
, 0xa, 0xf, false);
521 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(), dpp_row_bcast31
, 0xc, 0xf, false);
523 case aco_opcode::p_exclusive_scan
:
524 if (ctx
->program
->chip_class
>= GFX10
) { /* gfx10 doesn't support wf_sr1, so emulate it */
525 /* shift rows right */
526 emit_dpp_mov(ctx
, vtmp
, tmp
, src
.size(), dpp_row_sr(1), 0xf, 0xf, true);
528 /* fill in the gaps in rows 1 and 3 */
529 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_lo
, s1
), Operand(0x10000u
));
530 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_hi
, s1
), Operand(0x10000u
));
531 for (unsigned i
= 0; i
< src
.size(); i
++) {
532 Instruction
*perm
= bld
.vop3(aco_opcode::v_permlanex16_b32
,
533 Definition(PhysReg
{vtmp
+i
}, v1
),
534 Operand(PhysReg
{tmp
+i
}, v1
),
535 Operand(0xffffffffu
), Operand(0xffffffffu
)).instr
;
536 static_cast<VOP3A_instruction
*>(perm
)->opsel
= 1; /* FI (Fetch Inactive) */
538 bld
.sop1(Builder::s_mov
, Definition(exec
, bld
.lm
), Operand(UINT64_MAX
));
540 if (ctx
->program
->wave_size
== 64) {
541 /* fill in the gap in row 2 */
542 for (unsigned i
= 0; i
< src
.size(); i
++) {
543 bld
.readlane(Definition(PhysReg
{sitmp
+i
}, s1
), Operand(PhysReg
{tmp
+i
}, v1
), Operand(31u));
544 bld
.writelane(Definition(PhysReg
{vtmp
+i
}, v1
), Operand(PhysReg
{sitmp
+i
}, s1
), Operand(32u), Operand(PhysReg
{vtmp
+i
}, v1
));
547 std::swap(tmp
, vtmp
);
548 } else if (ctx
->program
->chip_class
>= GFX8
) {
549 emit_dpp_mov(ctx
, tmp
, tmp
, src
.size(), dpp_wf_sr1
, 0xf, 0xf, true);
551 // TODO: use LDS on CS with a single write and shifted read
552 /* wavefront shift_right by 1 on SI/CI */
553 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), (1 << 15) | dpp_quad_perm(0, 0, 1, 2));
554 emit_ds_swizzle(bld
, tmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1F, 0x00, 0x07)); /* mirror(8) */
555 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_lo
, s1
), Operand(0x10101010u
));
556 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_hi
, s1
), Operand(exec_lo
, s1
));
557 for (unsigned i
= 0; i
< src
.size(); i
++)
558 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{vtmp
+i
}, v1
), Operand(PhysReg
{tmp
+i
}, v1
));
560 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(UINT64_MAX
));
561 emit_ds_swizzle(bld
, tmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1F, 0x00, 0x08)); /* swap(8) */
562 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_lo
, s1
), Operand(0x01000100u
));
563 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_hi
, s1
), Operand(exec_lo
, s1
));
564 for (unsigned i
= 0; i
< src
.size(); i
++)
565 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{vtmp
+i
}, v1
), Operand(PhysReg
{tmp
+i
}, v1
));
567 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(UINT64_MAX
));
568 emit_ds_swizzle(bld
, tmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1F, 0x00, 0x10)); /* swap(16) */
569 bld
.sop2(aco_opcode::s_bfm_b32
, Definition(exec_lo
, s1
), Operand(1u), Operand(16u));
570 bld
.sop2(aco_opcode::s_bfm_b32
, Definition(exec_hi
, s1
), Operand(1u), Operand(16u));
571 for (unsigned i
= 0; i
< src
.size(); i
++)
572 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{vtmp
+i
}, v1
), Operand(PhysReg
{tmp
+i
}, v1
));
574 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(UINT64_MAX
));
575 for (unsigned i
= 0; i
< src
.size(); i
++) {
576 bld
.writelane(Definition(PhysReg
{vtmp
+i
}, v1
), identity
[i
], Operand(0u), Operand(PhysReg
{vtmp
+i
}, v1
));
577 bld
.readlane(Definition(PhysReg
{sitmp
+i
}, s1
), Operand(PhysReg
{tmp
+i
}, v1
), Operand(0u));
578 bld
.writelane(Definition(PhysReg
{vtmp
+i
}, v1
), Operand(PhysReg
{sitmp
+i
}, s1
), Operand(32u), Operand(PhysReg
{vtmp
+i
}, v1
));
579 identity
[i
] = Operand(0u); /* prevent further uses of identity */
581 std::swap(tmp
, vtmp
);
584 for (unsigned i
= 0; i
< src
.size(); i
++) {
585 if (!identity
[i
].isConstant() || identity
[i
].constantValue()) { /* bound_ctrl should take care of this overwise */
586 if (ctx
->program
->chip_class
< GFX10
)
587 assert((identity
[i
].isConstant() && !identity
[i
].isLiteral()) || identity
[i
].physReg() == PhysReg
{sitmp
+i
});
588 bld
.writelane(Definition(PhysReg
{tmp
+i
}, v1
), identity
[i
], Operand(0u), Operand(PhysReg
{tmp
+i
}, v1
));
592 case aco_opcode::p_inclusive_scan
:
593 assert(cluster_size
== ctx
->program
->wave_size
);
594 if (ctx
->program
->chip_class
<= GFX7
) {
595 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1e, 0x00, 0x00));
596 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_lo
, s1
), Operand(0xAAAAAAAAu
));
597 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_hi
, s1
), Operand(exec_lo
, s1
));
598 emit_op(ctx
, tmp
, tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
600 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(UINT64_MAX
));
601 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x1c, 0x01, 0x00));
602 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_lo
, s1
), Operand(0xCCCCCCCCu
));
603 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_hi
, s1
), Operand(exec_lo
, s1
));
604 emit_op(ctx
, tmp
, tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
606 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(UINT64_MAX
));
607 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x18, 0x03, 0x00));
608 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_lo
, s1
), Operand(0xF0F0F0F0u
));
609 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_hi
, s1
), Operand(exec_lo
, s1
));
610 emit_op(ctx
, tmp
, tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
612 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(UINT64_MAX
));
613 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x10, 0x07, 0x00));
614 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_lo
, s1
), Operand(0xFF00FF00u
));
615 bld
.sop1(aco_opcode::s_mov_b32
, Definition(exec_hi
, s1
), Operand(exec_lo
, s1
));
616 emit_op(ctx
, tmp
, tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
618 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(UINT64_MAX
));
619 emit_ds_swizzle(bld
, vtmp
, tmp
, src
.size(), ds_pattern_bitmode(0x00, 0x0f, 0x00));
620 bld
.sop2(aco_opcode::s_bfm_b32
, Definition(exec_lo
, s1
), Operand(16u), Operand(16u));
621 bld
.sop2(aco_opcode::s_bfm_b32
, Definition(exec_hi
, s1
), Operand(16u), Operand(16u));
622 emit_op(ctx
, tmp
, tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
624 for (unsigned i
= 0; i
< src
.size(); i
++)
625 bld
.readlane(Definition(PhysReg
{sitmp
+i
}, s1
), Operand(PhysReg
{tmp
+i
}, v1
), Operand(31u));
626 bld
.sop2(aco_opcode::s_bfm_b64
, Definition(exec
, s2
), Operand(32u), Operand(32u));
627 emit_op(ctx
, tmp
, sitmp
, tmp
, vtmp
, reduce_op
, src
.size());
631 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(),
632 dpp_row_sr(1), 0xf, 0xf, false, identity
);
633 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(),
634 dpp_row_sr(2), 0xf, 0xf, false, identity
);
635 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(),
636 dpp_row_sr(4), 0xf, 0xf, false, identity
);
637 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(),
638 dpp_row_sr(8), 0xf, 0xf, false, identity
);
639 if (ctx
->program
->chip_class
>= GFX10
) {
640 bld
.sop2(aco_opcode::s_bfm_b32
, Definition(exec_lo
, s1
), Operand(16u), Operand(16u));
641 bld
.sop2(aco_opcode::s_bfm_b32
, Definition(exec_hi
, s1
), Operand(16u), Operand(16u));
642 for (unsigned i
= 0; i
< src
.size(); i
++) {
643 Instruction
*perm
= bld
.vop3(aco_opcode::v_permlanex16_b32
,
644 Definition(PhysReg
{vtmp
+i
}, v1
),
645 Operand(PhysReg
{tmp
+i
}, v1
),
646 Operand(0xffffffffu
), Operand(0xffffffffu
)).instr
;
647 static_cast<VOP3A_instruction
*>(perm
)->opsel
= 1; /* FI (Fetch Inactive) */
649 emit_op(ctx
, tmp
, tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
651 if (ctx
->program
->wave_size
== 64) {
652 bld
.sop2(aco_opcode::s_bfm_b64
, Definition(exec
, s2
), Operand(32u), Operand(32u));
653 for (unsigned i
= 0; i
< src
.size(); i
++)
654 bld
.readlane(Definition(PhysReg
{sitmp
+i
}, s1
), Operand(PhysReg
{tmp
+i
}, v1
), Operand(31u));
655 emit_op(ctx
, tmp
, sitmp
, tmp
, vtmp
, reduce_op
, src
.size());
658 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(),
659 dpp_row_bcast15
, 0xa, 0xf, false, identity
);
660 emit_dpp_op(ctx
, tmp
, tmp
, tmp
, vtmp
, reduce_op
, src
.size(),
661 dpp_row_bcast31
, 0xc, 0xf, false, identity
);
665 unreachable("Invalid reduction mode");
669 if (op
== aco_opcode::p_reduce
) {
670 if (reduction_needs_last_op
&& dst
.regClass().type() == RegType::vgpr
) {
671 bld
.sop1(Builder::s_mov
, Definition(exec
, bld
.lm
), Operand(stmp
, bld
.lm
));
672 emit_op(ctx
, dst
.physReg(), tmp
, vtmp
, PhysReg
{0}, reduce_op
, src
.size());
676 if (reduction_needs_last_op
)
677 emit_op(ctx
, tmp
, vtmp
, tmp
, PhysReg
{0}, reduce_op
, src
.size());
681 bld
.sop1(Builder::s_mov
, Definition(exec
, bld
.lm
), Operand(stmp
, bld
.lm
));
683 if (dst
.regClass().type() == RegType::sgpr
) {
684 for (unsigned k
= 0; k
< src
.size(); k
++) {
685 bld
.readlane(Definition(PhysReg
{dst
.physReg() + k
}, s1
),
686 Operand(PhysReg
{tmp
+ k
}, v1
), Operand(ctx
->program
->wave_size
- 1));
688 } else if (dst
.physReg() != tmp
) {
689 for (unsigned k
= 0; k
< src
.size(); k
++) {
690 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{dst
.physReg() + k
}, v1
),
691 Operand(PhysReg
{tmp
+ k
}, v1
));
696 struct copy_operation
{
702 uint64_t is_used
= 0;
706 void handle_operands(std::map
<PhysReg
, copy_operation
>& copy_map
, lower_context
* ctx
, chip_class chip_class
, Pseudo_instruction
*pi
)
708 Builder
bld(ctx
->program
, &ctx
->instructions
);
709 aco_ptr
<Instruction
> mov
;
710 std::map
<PhysReg
, copy_operation
>::iterator it
= copy_map
.begin();
711 std::map
<PhysReg
, copy_operation
>::iterator target
;
712 bool writes_scc
= false;
714 /* count the number of uses for each dst reg */
715 while (it
!= copy_map
.end()) {
717 if (it
->second
.def
.physReg() == scc
)
720 assert(!pi
->tmp_in_scc
|| !(it
->second
.def
.physReg() == pi
->scratch_sgpr
));
722 /* if src and dst reg are the same, remove operation */
723 if (it
->first
== it
->second
.op
.physReg()) {
724 it
= copy_map
.erase(it
);
728 /* split large copies */
729 if (it
->second
.bytes
> 8) {
730 assert(!it
->second
.op
.isConstant());
731 assert(!it
->second
.def
.regClass().is_subdword());
732 RegClass rc
= RegClass(it
->second
.def
.regClass().type(), it
->second
.def
.size() - 2);
733 Definition hi_def
= Definition(PhysReg
{it
->first
+ 2}, rc
);
734 rc
= RegClass(it
->second
.op
.regClass().type(), it
->second
.op
.size() - 2);
735 Operand hi_op
= Operand(PhysReg
{it
->second
.op
.physReg() + 2}, rc
);
736 copy_operation copy
= {hi_op
, hi_def
, it
->second
.bytes
- 8};
737 copy_map
[hi_def
.physReg()] = copy
;
738 assert(it
->second
.op
.physReg().byte() == 0 && it
->second
.def
.physReg().byte() == 0);
739 it
->second
.op
= Operand(it
->second
.op
.physReg(), it
->second
.op
.regClass().type() == RegType::sgpr
? s2
: v2
);
740 it
->second
.def
= Definition(it
->second
.def
.physReg(), it
->second
.def
.regClass().type() == RegType::sgpr
? s2
: v2
);
741 it
->second
.bytes
= 8;
744 /* check if the definition reg is used by another copy operation */
745 for (std::pair
<const PhysReg
, copy_operation
>& copy
: copy_map
) {
746 if (copy
.second
.op
.isConstant())
748 for (uint16_t i
= 0; i
< it
->second
.bytes
; i
++) {
749 /* distance might underflow */
750 unsigned distance
= it
->first
.reg_b
+ i
- copy
.second
.op
.physReg().reg_b
;
751 if (distance
< copy
.second
.bytes
)
752 it
->second
.uses
[i
] += 1;
759 /* first, handle paths in the location transfer graph */
760 bool preserve_scc
= pi
->tmp_in_scc
&& !writes_scc
;
761 it
= copy_map
.begin();
762 while (it
!= copy_map
.end()) {
764 /* split cross half-reg copies: SDWA can only access bytes and shorts */
765 if (it
->second
.def
.regClass().is_subdword()) {
766 PhysReg def_reg
= it
->second
.def
.physReg();
767 PhysReg op_reg
= it
->second
.op
.physReg();
768 unsigned new_bytes
= 0;
769 if (it
->second
.bytes
> 1 && (def_reg
.byte() % 2 || op_reg
.byte() % 2)) {
771 } else if (it
->second
.bytes
> 2 && (def_reg
.byte() || op_reg
.byte())) {
773 } else if (it
->second
.bytes
== 3) {
775 } else if (it
->second
.bytes
> 4) {
776 assert(it
->second
.op
.physReg().byte() == 0 && it
->second
.def
.physReg().byte() == 0);
780 RegClass rc
= RegClass(RegType::vgpr
, it
->second
.bytes
- new_bytes
).as_subdword();
781 def_reg
.reg_b
+= new_bytes
;
782 op_reg
.reg_b
+= new_bytes
;
783 copy_operation copy
= {Operand(op_reg
, rc
), Definition(def_reg
, rc
), it
->second
.bytes
- new_bytes
};
784 copy
.is_used
= it
->second
.is_used
>> (8 * new_bytes
);
785 copy_map
[def_reg
] = copy
;
786 rc
= RegClass(RegType::vgpr
, new_bytes
).as_subdword();
787 it
->second
.op
= Operand(it
->second
.op
.physReg(), rc
);
788 it
->second
.def
= Definition(it
->second
.def
.physReg(), rc
);
789 it
->second
.is_used
= it
->second
.is_used
& ((1 << (8 * new_bytes
)) - 1);
790 it
->second
.bytes
= new_bytes
;
793 /* convert dword moves to normal regclass */
794 if (it
->second
.bytes
== 4) {
795 it
->second
.op
= Operand(it
->second
.op
.physReg(), v1
);
796 it
->second
.def
= Definition(it
->second
.def
.physReg(), v1
);
800 /* split multi-reg copies */
801 if (it
->second
.bytes
> 4 && !it
->second
.op
.isConstant()) {
802 assert(!it
->second
.def
.regClass().is_subdword());
803 RegClass rc
= RegClass(it
->second
.def
.regClass().type(), it
->second
.def
.size() - 1);
804 Definition hi_def
= Definition(PhysReg
{it
->first
+ 1}, rc
);
805 rc
= RegClass(it
->second
.op
.regClass().type(), it
->second
.op
.size() - 1);
806 Operand hi_op
= Operand(PhysReg
{it
->second
.op
.physReg() + 1}, rc
);
807 copy_operation copy
= {hi_op
, hi_def
, it
->second
.bytes
- 4};
808 copy
.is_used
= it
->second
.is_used
>> 32;
809 copy_map
[hi_def
.physReg()] = copy
;
810 assert(it
->second
.op
.physReg().byte() == 0 && it
->second
.def
.physReg().byte() == 0);
811 it
->second
.op
= Operand(it
->second
.op
.physReg(), it
->second
.op
.regClass().type() == RegType::sgpr
? s1
: v1
);
812 it
->second
.def
= Definition(it
->second
.def
.physReg(), it
->second
.def
.regClass().type() == RegType::sgpr
? s1
: v1
);
813 it
->second
.is_used
= it
->second
.is_used
& 0xFFFFFFFF;
814 it
->second
.bytes
= 4;
817 /* the target reg is not used as operand for any other copy */
818 if (it
->second
.is_used
== 0) {
820 /* try to coalesce 32-bit sgpr copies to 64-bit copies */
821 if (it
->second
.def
.getTemp().type() == RegType::sgpr
&& it
->second
.bytes
== 4 &&
822 !it
->second
.op
.isConstant() && it
->first
% 2 == it
->second
.op
.physReg() % 2) {
824 PhysReg other_def_reg
= PhysReg
{it
->first
% 2 ? it
->first
- 1 : it
->first
+ 1};
825 PhysReg other_op_reg
= PhysReg
{it
->first
% 2 ? it
->second
.op
.physReg() - 1 : it
->second
.op
.physReg() + 1};
826 std::map
<PhysReg
, copy_operation
>::iterator other
= copy_map
.find(other_def_reg
);
828 if (other
!= copy_map
.end() && !other
->second
.is_used
&& other
->second
.bytes
== 4 &&
829 other
->second
.op
.physReg() == other_op_reg
&& !other
->second
.op
.isConstant()) {
830 std::map
<PhysReg
, copy_operation
>::iterator to_erase
= it
->first
% 2 ? it
: other
;
831 it
= it
->first
% 2 ? other
: it
;
832 copy_map
.erase(to_erase
);
833 it
->second
.bytes
= 8;
836 // TODO: try to coalesce subdword copies
838 if (it
->second
.def
.physReg() == scc
) {
839 bld
.sopc(aco_opcode::s_cmp_lg_i32
, it
->second
.def
, it
->second
.op
, Operand(0u));
841 } else if (it
->second
.bytes
== 8 && it
->second
.def
.getTemp().type() == RegType::sgpr
) {
842 bld
.sop1(aco_opcode::s_mov_b64
, it
->second
.def
, Operand(it
->second
.op
.physReg(), s2
));
843 } else if (it
->second
.bytes
== 8 && it
->second
.op
.isConstant()) {
844 uint64_t val
= it
->second
.op
.constantValue64();
845 bld
.vop1(aco_opcode::v_mov_b32
, it
->second
.def
, Operand((uint32_t)val
));
846 bld
.vop1(aco_opcode::v_mov_b32
, Definition(PhysReg
{it
->second
.def
.physReg() + 1}, v1
),
847 Operand((uint32_t)(val
>> 32)));
848 ctx
->program
->statistics
[statistic_copies
]++;
850 bld
.copy(it
->second
.def
, it
->second
.op
);
853 /* reduce the number of uses of the operand reg by one */
854 if (!it
->second
.op
.isConstant()) {
855 for (std::pair
<const PhysReg
, copy_operation
>& copy
: copy_map
) {
856 for (uint16_t i
= 0; i
< copy
.second
.bytes
; i
++) {
857 /* distance might underflow */
858 unsigned distance
= copy
.first
.reg_b
+ i
- it
->second
.op
.physReg().reg_b
;
859 if (distance
< it
->second
.bytes
)
860 copy
.second
.uses
[i
] -= 1;
866 it
= copy_map
.begin();
867 ctx
->program
->statistics
[statistic_copies
]++;
870 /* the target reg is used as operand, check the next entry */
875 if (copy_map
.empty())
878 /* all target regs are needed as operand somewhere which means, all entries are part of a cycle */
879 for (it
= copy_map
.begin(); it
!= copy_map
.end(); ++it
) {
880 assert(it
->second
.op
.isFixed());
881 if (it
->first
== it
->second
.op
.physReg())
884 /* should already be done */
885 assert(!it
->second
.op
.isConstant());
887 if (preserve_scc
&& it
->second
.def
.getTemp().type() == RegType::sgpr
)
888 assert(!(it
->second
.def
.physReg() == pi
->scratch_sgpr
));
890 /* to resolve the cycle, we have to swap the src reg with the dst reg */
891 copy_operation swap
= it
->second
;
892 assert(swap
.op
.regClass() == swap
.def
.regClass());
893 Operand def_as_op
= Operand(swap
.def
.physReg(), swap
.def
.regClass());
894 Definition op_as_def
= Definition(swap
.op
.physReg(), swap
.op
.regClass());
895 if (chip_class
>= GFX9
&& swap
.def
.regClass() == v1
) {
896 bld
.vop1(aco_opcode::v_swap_b32
, swap
.def
, op_as_def
, swap
.op
, def_as_op
);
897 ctx
->program
->statistics
[statistic_copies
]++;
898 } else if (swap
.def
.regClass() == v1
) {
899 bld
.vop2(aco_opcode::v_xor_b32
, op_as_def
, swap
.op
, def_as_op
);
900 bld
.vop2(aco_opcode::v_xor_b32
, swap
.def
, swap
.op
, def_as_op
);
901 bld
.vop2(aco_opcode::v_xor_b32
, op_as_def
, swap
.op
, def_as_op
);
902 ctx
->program
->statistics
[statistic_copies
] += 3;
903 } else if (swap
.op
.physReg() == scc
|| swap
.def
.physReg() == scc
) {
904 /* we need to swap scc and another sgpr */
905 assert(!preserve_scc
);
907 PhysReg other
= swap
.op
.physReg() == scc
? swap
.def
.physReg() : swap
.op
.physReg();
909 bld
.sop1(aco_opcode::s_mov_b32
, Definition(pi
->scratch_sgpr
, s1
), Operand(scc
, s1
));
910 bld
.sopc(aco_opcode::s_cmp_lg_i32
, Definition(scc
, s1
), Operand(other
, s1
), Operand(0u));
911 bld
.sop1(aco_opcode::s_mov_b32
, Definition(other
, s1
), Operand(pi
->scratch_sgpr
, s1
));
912 ctx
->program
->statistics
[statistic_copies
] += 3;
913 } else if (swap
.def
.regClass() == s1
) {
915 bld
.sop1(aco_opcode::s_mov_b32
, Definition(pi
->scratch_sgpr
, s1
), swap
.op
);
916 bld
.sop1(aco_opcode::s_mov_b32
, op_as_def
, def_as_op
);
917 bld
.sop1(aco_opcode::s_mov_b32
, swap
.def
, Operand(pi
->scratch_sgpr
, s1
));
919 bld
.sop2(aco_opcode::s_xor_b32
, op_as_def
, Definition(scc
, s1
), swap
.op
, def_as_op
);
920 bld
.sop2(aco_opcode::s_xor_b32
, swap
.def
, Definition(scc
, s1
), swap
.op
, def_as_op
);
921 bld
.sop2(aco_opcode::s_xor_b32
, op_as_def
, Definition(scc
, s1
), swap
.op
, def_as_op
);
923 ctx
->program
->statistics
[statistic_copies
] += 3;
925 bld
.vop2(aco_opcode::v_xor_b32
, op_as_def
, swap
.op
, def_as_op
);
926 bld
.vop2(aco_opcode::v_xor_b32
, swap
.def
, swap
.op
, def_as_op
);
927 bld
.vop2(aco_opcode::v_xor_b32
, op_as_def
, swap
.op
, def_as_op
);
928 ctx
->program
->statistics
[statistic_copies
] += 3;
929 assert(swap
.def
.regClass().is_subdword());
930 assert(false && "Subdword swaps not yet implemented.");
933 /* change the operand reg of the target's use */
934 assert(swap
.is_used
== 0x01010101lu
); // each 1 use per byte
936 for (++target
; target
!= copy_map
.end(); ++target
) {
937 if (target
->second
.op
.physReg() == it
->first
) {
938 target
->second
.op
.setFixed(swap
.op
.physReg());
945 void lower_to_hw_instr(Program
* program
)
947 Block
*discard_block
= NULL
;
949 for (size_t i
= 0; i
< program
->blocks
.size(); i
++)
951 Block
*block
= &program
->blocks
[i
];
953 ctx
.program
= program
;
954 Builder
bld(program
, &ctx
.instructions
);
956 bool set_mode
= i
== 0 && block
->fp_mode
.val
!= program
->config
->float_mode
;
957 for (unsigned pred
: block
->linear_preds
) {
958 if (program
->blocks
[pred
].fp_mode
.val
!= block
->fp_mode
.val
) {
964 /* only allow changing modes at top-level blocks so this doesn't break
965 * the "jump over empty blocks" optimization */
966 assert(block
->kind
& block_kind_top_level
);
967 uint32_t mode
= block
->fp_mode
.val
;
968 /* "((size - 1) << 11) | register" (MODE is encoded as register 1) */
969 bld
.sopk(aco_opcode::s_setreg_imm32_b32
, Operand(mode
), (7 << 11) | 1);
972 for (size_t j
= 0; j
< block
->instructions
.size(); j
++) {
973 aco_ptr
<Instruction
>& instr
= block
->instructions
[j
];
974 aco_ptr
<Instruction
> mov
;
975 if (instr
->format
== Format::PSEUDO
) {
976 Pseudo_instruction
*pi
= (Pseudo_instruction
*)instr
.get();
978 switch (instr
->opcode
)
980 case aco_opcode::p_extract_vector
:
982 PhysReg reg
= instr
->operands
[0].physReg();
983 Definition
& def
= instr
->definitions
[0];
984 reg
.reg_b
+= instr
->operands
[1].constantValue() * def
.bytes();
986 if (reg
== def
.physReg())
989 RegClass op_rc
= def
.regClass().is_subdword() ? def
.regClass() :
990 RegClass(instr
->operands
[0].getTemp().type(), def
.size());
991 std::map
<PhysReg
, copy_operation
> copy_operations
;
992 copy_operations
[def
.physReg()] = {Operand(reg
, op_rc
), def
, def
.bytes()};
993 handle_operands(copy_operations
, &ctx
, program
->chip_class
, pi
);
996 case aco_opcode::p_create_vector
:
998 std::map
<PhysReg
, copy_operation
> copy_operations
;
999 PhysReg reg
= instr
->definitions
[0].physReg();
1001 for (const Operand
& op
: instr
->operands
) {
1002 if (op
.isConstant()) {
1003 const Definition def
= Definition(reg
, RegClass(instr
->definitions
[0].getTemp().type(), op
.size()));
1004 copy_operations
[reg
] = {op
, def
, op
.bytes()};
1005 reg
.reg_b
+= op
.bytes();
1008 if (op
.isUndefined()) {
1009 // TODO: coalesce subdword copies if dst byte is 0
1010 reg
.reg_b
+= op
.bytes();
1014 RegClass rc_def
= op
.regClass().is_subdword() ? op
.regClass() :
1015 RegClass(instr
->definitions
[0].getTemp().type(), op
.size());
1016 const Definition def
= Definition(reg
, rc_def
);
1017 copy_operations
[def
.physReg()] = {op
, def
, op
.bytes()};
1018 reg
.reg_b
+= op
.bytes();
1020 handle_operands(copy_operations
, &ctx
, program
->chip_class
, pi
);
1023 case aco_opcode::p_split_vector
:
1025 std::map
<PhysReg
, copy_operation
> copy_operations
;
1026 PhysReg reg
= instr
->operands
[0].physReg();
1028 for (const Definition
& def
: instr
->definitions
) {
1029 RegClass rc_op
= def
.regClass().is_subdword() ? def
.regClass() :
1030 RegClass(instr
->operands
[0].getTemp().type(), def
.size());
1031 const Operand op
= Operand(reg
, rc_op
);
1032 copy_operations
[def
.physReg()] = {op
, def
, def
.bytes()};
1033 reg
.reg_b
+= def
.bytes();
1035 handle_operands(copy_operations
, &ctx
, program
->chip_class
, pi
);
1038 case aco_opcode::p_parallelcopy
:
1039 case aco_opcode::p_wqm
:
1041 std::map
<PhysReg
, copy_operation
> copy_operations
;
1042 for (unsigned i
= 0; i
< instr
->operands
.size(); i
++) {
1043 assert(instr
->definitions
[i
].bytes() == instr
->operands
[i
].bytes());
1044 copy_operations
[instr
->definitions
[i
].physReg()] = {instr
->operands
[i
], instr
->definitions
[i
], instr
->operands
[i
].bytes()};
1046 handle_operands(copy_operations
, &ctx
, program
->chip_class
, pi
);
1049 case aco_opcode::p_exit_early_if
:
1051 /* don't bother with an early exit near the end of the program */
1052 if ((block
->instructions
.size() - 1 - j
) <= 4 &&
1053 block
->instructions
.back()->opcode
== aco_opcode::s_endpgm
) {
1054 unsigned null_exp_dest
= (ctx
.program
->stage
& hw_fs
) ? 9 /* NULL */ : V_008DFC_SQ_EXP_POS
;
1055 bool ignore_early_exit
= true;
1057 for (unsigned k
= j
+ 1; k
< block
->instructions
.size(); ++k
) {
1058 const aco_ptr
<Instruction
> &instr
= block
->instructions
[k
];
1059 if (instr
->opcode
== aco_opcode::s_endpgm
||
1060 instr
->opcode
== aco_opcode::p_logical_end
)
1062 else if (instr
->opcode
== aco_opcode::exp
&&
1063 static_cast<Export_instruction
*>(instr
.get())->dest
== null_exp_dest
)
1065 else if (instr
->opcode
== aco_opcode::p_parallelcopy
&&
1066 instr
->definitions
[0].isFixed() &&
1067 instr
->definitions
[0].physReg() == exec
)
1070 ignore_early_exit
= false;
1073 if (ignore_early_exit
)
1077 if (!discard_block
) {
1078 discard_block
= program
->create_and_insert_block();
1079 block
= &program
->blocks
[i
];
1081 bld
.reset(discard_block
);
1082 bld
.exp(aco_opcode::exp
, Operand(v1
), Operand(v1
), Operand(v1
), Operand(v1
),
1083 0, V_008DFC_SQ_EXP_NULL
, false, true, true);
1084 if (program
->wb_smem_l1_on_end
)
1085 bld
.smem(aco_opcode::s_dcache_wb
);
1086 bld
.sopp(aco_opcode::s_endpgm
);
1088 bld
.reset(&ctx
.instructions
);
1091 //TODO: exec can be zero here with block_kind_discard
1093 assert(instr
->operands
[0].physReg() == scc
);
1094 bld
.sopp(aco_opcode::s_cbranch_scc0
, instr
->operands
[0], discard_block
->index
);
1096 discard_block
->linear_preds
.push_back(block
->index
);
1097 block
->linear_succs
.push_back(discard_block
->index
);
1100 case aco_opcode::p_spill
:
1102 assert(instr
->operands
[0].regClass() == v1
.as_linear());
1103 for (unsigned i
= 0; i
< instr
->operands
[2].size(); i
++)
1104 bld
.writelane(bld
.def(v1
, instr
->operands
[0].physReg()),
1105 Operand(PhysReg
{instr
->operands
[2].physReg() + i
}, s1
),
1106 Operand(instr
->operands
[1].constantValue() + i
),
1107 instr
->operands
[0]);
1110 case aco_opcode::p_reload
:
1112 assert(instr
->operands
[0].regClass() == v1
.as_linear());
1113 for (unsigned i
= 0; i
< instr
->definitions
[0].size(); i
++)
1114 bld
.readlane(bld
.def(s1
, PhysReg
{instr
->definitions
[0].physReg() + i
}),
1116 Operand(instr
->operands
[1].constantValue() + i
));
1119 case aco_opcode::p_as_uniform
:
1121 if (instr
->operands
[0].isConstant() || instr
->operands
[0].regClass().type() == RegType::sgpr
) {
1122 std::map
<PhysReg
, copy_operation
> copy_operations
;
1123 copy_operations
[instr
->definitions
[0].physReg()] = {instr
->operands
[0], instr
->definitions
[0], instr
->definitions
[0].bytes()};
1124 handle_operands(copy_operations
, &ctx
, program
->chip_class
, pi
);
1126 assert(instr
->operands
[0].regClass().type() == RegType::vgpr
);
1127 assert(instr
->definitions
[0].regClass().type() == RegType::sgpr
);
1128 assert(instr
->operands
[0].size() == instr
->definitions
[0].size());
1129 for (unsigned i
= 0; i
< instr
->definitions
[0].size(); i
++) {
1130 bld
.vop1(aco_opcode::v_readfirstlane_b32
,
1131 bld
.def(s1
, PhysReg
{instr
->definitions
[0].physReg() + i
}),
1132 Operand(PhysReg
{instr
->operands
[0].physReg() + i
}, v1
));
1140 } else if (instr
->format
== Format::PSEUDO_BRANCH
) {
1141 Pseudo_branch_instruction
* branch
= static_cast<Pseudo_branch_instruction
*>(instr
.get());
1142 /* check if all blocks from current to target are empty */
1143 bool can_remove
= block
->index
< branch
->target
[0];
1144 for (unsigned i
= block
->index
+ 1; can_remove
&& i
< branch
->target
[0]; i
++) {
1145 if (program
->blocks
[i
].instructions
.size())
1151 switch (instr
->opcode
) {
1152 case aco_opcode::p_branch
:
1153 assert(block
->linear_succs
[0] == branch
->target
[0]);
1154 bld
.sopp(aco_opcode::s_branch
, branch
->target
[0]);
1156 case aco_opcode::p_cbranch_nz
:
1157 assert(block
->linear_succs
[1] == branch
->target
[0]);
1158 if (branch
->operands
[0].physReg() == exec
)
1159 bld
.sopp(aco_opcode::s_cbranch_execnz
, branch
->target
[0]);
1160 else if (branch
->operands
[0].physReg() == vcc
)
1161 bld
.sopp(aco_opcode::s_cbranch_vccnz
, branch
->target
[0]);
1163 assert(branch
->operands
[0].physReg() == scc
);
1164 bld
.sopp(aco_opcode::s_cbranch_scc1
, branch
->target
[0]);
1167 case aco_opcode::p_cbranch_z
:
1168 assert(block
->linear_succs
[1] == branch
->target
[0]);
1169 if (branch
->operands
[0].physReg() == exec
)
1170 bld
.sopp(aco_opcode::s_cbranch_execz
, branch
->target
[0]);
1171 else if (branch
->operands
[0].physReg() == vcc
)
1172 bld
.sopp(aco_opcode::s_cbranch_vccz
, branch
->target
[0]);
1174 assert(branch
->operands
[0].physReg() == scc
);
1175 bld
.sopp(aco_opcode::s_cbranch_scc0
, branch
->target
[0]);
1179 unreachable("Unknown Pseudo branch instruction!");
1182 } else if (instr
->format
== Format::PSEUDO_REDUCTION
) {
1183 Pseudo_reduction_instruction
* reduce
= static_cast<Pseudo_reduction_instruction
*>(instr
.get());
1184 if (reduce
->reduce_op
== gfx10_wave64_bpermute
) {
1185 /* Only makes sense on GFX10 wave64 */
1186 assert(program
->chip_class
>= GFX10
);
1187 assert(program
->info
->wave_size
== 64);
1188 assert(instr
->definitions
[0].regClass() == v1
); /* Destination */
1189 assert(instr
->definitions
[1].regClass() == s2
); /* Temp EXEC */
1190 assert(instr
->definitions
[1].physReg() != vcc
);
1191 assert(instr
->definitions
[2].physReg() == scc
); /* SCC clobber */
1192 assert(instr
->operands
[0].physReg() == vcc
); /* Compare */
1193 assert(instr
->operands
[1].regClass() == v2
.as_linear()); /* Temp VGPR pair */
1194 assert(instr
->operands
[2].regClass() == v1
); /* Indices x4 */
1195 assert(instr
->operands
[3].regClass() == v1
); /* Input data */
1197 PhysReg shared_vgpr_reg_lo
= PhysReg(align(program
->config
->num_vgprs
, 4) + 256);
1198 PhysReg shared_vgpr_reg_hi
= PhysReg(shared_vgpr_reg_lo
+ 1);
1199 Operand compare
= instr
->operands
[0];
1200 Operand
tmp1(instr
->operands
[1].physReg(), v1
);
1201 Operand
tmp2(PhysReg(instr
->operands
[1].physReg() + 1), v1
);
1202 Operand index_x4
= instr
->operands
[2];
1203 Operand input_data
= instr
->operands
[3];
1204 Definition
shared_vgpr_lo(shared_vgpr_reg_lo
, v1
);
1205 Definition
shared_vgpr_hi(shared_vgpr_reg_hi
, v1
);
1206 Definition
def_temp1(tmp1
.physReg(), v1
);
1207 Definition
def_temp2(tmp2
.physReg(), v1
);
1209 /* Save EXEC and set it for all lanes */
1210 bld
.sop1(aco_opcode::s_or_saveexec_b64
, instr
->definitions
[1], instr
->definitions
[2],
1211 Definition(exec
, s2
), Operand((uint64_t)-1), Operand(exec
, s2
));
1213 /* HI: Copy data from high lanes 32-63 to shared vgpr */
1214 bld
.vop1_dpp(aco_opcode::v_mov_b32
, shared_vgpr_hi
, input_data
, dpp_quad_perm(0, 1, 2, 3), 0xc, 0xf, false);
1216 /* LO: Copy data from low lanes 0-31 to shared vgpr */
1217 bld
.vop1_dpp(aco_opcode::v_mov_b32
, shared_vgpr_lo
, input_data
, dpp_quad_perm(0, 1, 2, 3), 0x3, 0xf, false);
1218 /* LO: Copy shared vgpr (high lanes' data) to output vgpr */
1219 bld
.vop1_dpp(aco_opcode::v_mov_b32
, def_temp1
, Operand(shared_vgpr_reg_hi
, v1
), dpp_quad_perm(0, 1, 2, 3), 0x3, 0xf, false);
1221 /* HI: Copy shared vgpr (low lanes' data) to output vgpr */
1222 bld
.vop1_dpp(aco_opcode::v_mov_b32
, def_temp1
, Operand(shared_vgpr_reg_lo
, v1
), dpp_quad_perm(0, 1, 2, 3), 0xc, 0xf, false);
1224 /* Permute the original input */
1225 bld
.ds(aco_opcode::ds_bpermute_b32
, def_temp2
, index_x4
, input_data
);
1226 /* Permute the swapped input */
1227 bld
.ds(aco_opcode::ds_bpermute_b32
, def_temp1
, index_x4
, tmp1
);
1229 /* Restore saved EXEC */
1230 bld
.sop1(aco_opcode::s_mov_b64
, Definition(exec
, s2
), Operand(instr
->definitions
[1].physReg(), s2
));
1231 /* Choose whether to use the original or swapped */
1232 bld
.vop2(aco_opcode::v_cndmask_b32
, instr
->definitions
[0], tmp1
, tmp2
, compare
);
1234 emit_reduction(&ctx
, reduce
->opcode
, reduce
->reduce_op
, reduce
->cluster_size
,
1235 reduce
->operands
[1].physReg(), // tmp
1236 reduce
->definitions
[1].physReg(), // stmp
1237 reduce
->operands
[2].physReg(), // vtmp
1238 reduce
->definitions
[2].physReg(), // sitmp
1239 reduce
->operands
[0], reduce
->definitions
[0]);
1242 ctx
.instructions
.emplace_back(std::move(instr
));
1246 block
->instructions
.swap(ctx
.instructions
);