2 * Copyright © 2014 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Jason Ekstrand (jason@jlekstrand.net)
29 #include "compiler/nir/nir_builder.h"
32 * Implements a small peephole optimization that looks for a multiply that
33 * is only ever used in an add and replaces both with an fma.
37 are_all_uses_fadd(nir_ssa_def
*def
)
39 if (!list_is_empty(&def
->if_uses
))
42 nir_foreach_use(use_src
, def
) {
43 nir_instr
*use_instr
= use_src
->parent_instr
;
45 if (use_instr
->type
!= nir_instr_type_alu
)
48 nir_alu_instr
*use_alu
= nir_instr_as_alu(use_instr
);
49 switch (use_alu
->op
) {
51 break; /* This one's ok */
56 assert(use_alu
->dest
.dest
.is_ssa
);
57 if (!are_all_uses_fadd(&use_alu
->dest
.dest
.ssa
))
69 static nir_alu_instr
*
70 get_mul_for_src(nir_alu_src
*src
, unsigned num_components
,
71 uint8_t swizzle
[4], bool *negate
, bool *abs
)
73 uint8_t swizzle_tmp
[4];
74 assert(src
->src
.is_ssa
&& !src
->abs
&& !src
->negate
);
76 nir_instr
*instr
= src
->src
.ssa
->parent_instr
;
77 if (instr
->type
!= nir_instr_type_alu
)
80 nir_alu_instr
*alu
= nir_instr_as_alu(instr
);
82 /* We want to bail if any of the other ALU operations involved is labled
83 * exact. One reason for this is that, while the value that is changing is
84 * actually the result of the add and not the multiply, the intention of
85 * the user when they specify an exact multiply is that they want *that*
86 * value and what they don't care about is the add. Another reason is that
87 * SPIR-V explicitly requires this behaviour.
94 alu
= get_mul_for_src(&alu
->src
[0], alu
->dest
.dest
.ssa
.num_components
,
95 swizzle
, negate
, abs
);
99 alu
= get_mul_for_src(&alu
->src
[0], alu
->dest
.dest
.ssa
.num_components
,
100 swizzle
, negate
, abs
);
105 alu
= get_mul_for_src(&alu
->src
[0], alu
->dest
.dest
.ssa
.num_components
,
106 swizzle
, negate
, abs
);
112 /* Only absorb a fmul into a ffma if the fmul is only used in fadd
113 * operations. This prevents us from being too aggressive with our
114 * fusing which can actually lead to more instructions.
116 if (!are_all_uses_fadd(&alu
->dest
.dest
.ssa
))
127 /* Copy swizzle data before overwriting it to avoid setting a wrong swizzle.
130 * Former swizzle[] = xyzw
131 * src->swizzle[] = zyxx
133 * Expected output swizzle = zyxx
134 * If we reuse swizzle in the loop, then output swizzle would be zyzz.
136 memcpy(swizzle_tmp
, swizzle
, 4*sizeof(uint8_t));
137 for (int i
= 0; i
< num_components
; i
++)
138 swizzle
[i
] = swizzle_tmp
[src
->swizzle
[i
]];
144 * Given a list of (at least two) nir_alu_src's, tells if any of them is a
145 * constant value and is used only once.
148 any_alu_src_is_a_constant(nir_alu_src srcs
[])
150 for (unsigned i
= 0; i
< 2; i
++) {
151 if (srcs
[i
].src
.ssa
->parent_instr
->type
== nir_instr_type_load_const
) {
152 nir_load_const_instr
*load_const
=
153 nir_instr_as_load_const (srcs
[i
].src
.ssa
->parent_instr
);
155 if (list_is_singular(&load_const
->def
.uses
) &&
156 list_is_empty(&load_const
->def
.if_uses
)) {
166 brw_nir_opt_peephole_ffma_block(nir_builder
*b
, nir_block
*block
)
168 bool progress
= false;
170 nir_foreach_instr_safe(instr
, block
) {
171 if (instr
->type
!= nir_instr_type_alu
)
174 nir_alu_instr
*add
= nir_instr_as_alu(instr
);
175 if (add
->op
!= nir_op_fadd
)
178 assert(add
->dest
.dest
.is_ssa
);
182 assert(add
->src
[0].src
.is_ssa
&& add
->src
[1].src
.is_ssa
);
184 /* This, is the case a + a. We would rather handle this with an
185 * algebraic reduction than fuse it. Also, we want to only fuse
186 * things where the multiply is used only once and, in this case,
187 * it would be used twice by the same instruction.
189 if (add
->src
[0].src
.ssa
== add
->src
[1].src
.ssa
)
193 uint8_t add_mul_src
, swizzle
[4];
195 for (add_mul_src
= 0; add_mul_src
< 2; add_mul_src
++) {
196 for (unsigned i
= 0; i
< 4; i
++)
202 mul
= get_mul_for_src(&add
->src
[add_mul_src
],
203 add
->dest
.dest
.ssa
.num_components
,
204 swizzle
, &negate
, &abs
);
213 unsigned bit_size
= add
->dest
.dest
.ssa
.bit_size
;
215 nir_ssa_def
*mul_src
[2];
216 mul_src
[0] = mul
->src
[0].src
.ssa
;
217 mul_src
[1] = mul
->src
[1].src
.ssa
;
219 /* If any of the operands of the fmul and any of the fadd is a constant,
220 * we bypass because it will be more efficient as the constants will be
221 * propagated as operands, potentially saving two load_const instructions.
223 if (any_alu_src_is_a_constant(mul
->src
) &&
224 any_alu_src_is_a_constant(add
->src
)) {
228 b
->cursor
= nir_before_instr(&add
->instr
);
231 for (unsigned i
= 0; i
< 2; i
++)
232 mul_src
[i
] = nir_fabs(b
, mul_src
[i
]);
236 mul_src
[0] = nir_fneg(b
, mul_src
[0]);
238 nir_alu_instr
*ffma
= nir_alu_instr_create(b
->shader
, nir_op_ffma
);
239 ffma
->dest
.saturate
= add
->dest
.saturate
;
240 ffma
->dest
.write_mask
= add
->dest
.write_mask
;
242 for (unsigned i
= 0; i
< 2; i
++) {
243 ffma
->src
[i
].src
= nir_src_for_ssa(mul_src
[i
]);
244 for (unsigned j
= 0; j
< add
->dest
.dest
.ssa
.num_components
; j
++)
245 ffma
->src
[i
].swizzle
[j
] = mul
->src
[i
].swizzle
[swizzle
[j
]];
247 nir_alu_src_copy(&ffma
->src
[2], &add
->src
[1 - add_mul_src
], ffma
);
249 assert(add
->dest
.dest
.is_ssa
);
251 nir_ssa_dest_init(&ffma
->instr
, &ffma
->dest
.dest
,
252 add
->dest
.dest
.ssa
.num_components
,
254 add
->dest
.dest
.ssa
.name
);
255 nir_ssa_def_rewrite_uses(&add
->dest
.dest
.ssa
,
256 nir_src_for_ssa(&ffma
->dest
.dest
.ssa
));
258 nir_builder_instr_insert(b
, &ffma
->instr
);
259 assert(list_is_empty(&add
->dest
.dest
.ssa
.uses
));
260 nir_instr_remove(&add
->instr
);
269 brw_nir_opt_peephole_ffma_impl(nir_function_impl
*impl
)
271 bool progress
= false;
274 nir_builder_init(&builder
, impl
);
276 nir_foreach_block(block
, impl
) {
277 progress
|= brw_nir_opt_peephole_ffma_block(&builder
, block
);
281 nir_metadata_preserve(impl
, nir_metadata_block_index
|
282 nir_metadata_dominance
);
284 nir_metadata_preserve(impl
, nir_metadata_all
);
291 brw_nir_opt_peephole_ffma(nir_shader
*shader
)
293 bool progress
= false;
295 nir_foreach_function(function
, shader
) {
297 progress
|= brw_nir_opt_peephole_ffma_impl(function
->impl
);