2 * Copyright © 2014 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Jason Ekstrand (jason@jlekstrand.net)
31 * Implements a simple pass that lowers vecN instructions to a series of
32 * moves with partial writes.
35 struct vec_to_movs_state
{
36 nir_function_impl
*impl
;
41 src_matches_dest_reg(nir_dest
*dest
, nir_src
*src
)
43 if (dest
->is_ssa
|| src
->is_ssa
)
46 return (dest
->reg
.reg
== src
->reg
.reg
&&
47 dest
->reg
.base_offset
== src
->reg
.base_offset
&&
48 !dest
->reg
.indirect
&&
53 * For a given starting writemask channel and corresponding source index in
54 * the vec instruction, insert a MOV to the vec instruction's dest of all the
55 * writemask channels that get read from the same src reg.
57 * Returns the writemask of our MOV, so the parent loop calling this knows
58 * which ones have been processed.
61 insert_mov(nir_alu_instr
*vec
, unsigned start_idx
, nir_shader
*shader
)
63 assert(start_idx
< nir_op_infos
[vec
->op
].num_inputs
);
65 nir_alu_instr
*mov
= nir_alu_instr_create(shader
, nir_op_imov
);
66 nir_alu_src_copy(&mov
->src
[0], &vec
->src
[start_idx
], mov
);
67 nir_alu_dest_copy(&mov
->dest
, &vec
->dest
, mov
);
69 mov
->dest
.write_mask
= (1u << start_idx
);
70 mov
->src
[0].swizzle
[start_idx
] = vec
->src
[start_idx
].swizzle
[0];
71 mov
->src
[0].negate
= vec
->src
[start_idx
].negate
;
72 mov
->src
[0].abs
= vec
->src
[start_idx
].abs
;
74 for (unsigned i
= start_idx
+ 1; i
< 4; i
++) {
75 if (!(vec
->dest
.write_mask
& (1 << i
)))
78 if (nir_srcs_equal(vec
->src
[i
].src
, vec
->src
[start_idx
].src
) &&
79 vec
->src
[i
].negate
== vec
->src
[start_idx
].negate
&&
80 vec
->src
[i
].abs
== vec
->src
[start_idx
].abs
) {
81 mov
->dest
.write_mask
|= (1 << i
);
82 mov
->src
[0].swizzle
[i
] = vec
->src
[i
].swizzle
[0];
86 unsigned channels_handled
= mov
->dest
.write_mask
;
88 /* In some situations (if the vecN is involved in a phi-web), we can end
89 * up with a mov from a register to itself. Some of those channels may end
90 * up doing nothing and there's no reason to have them as part of the mov.
92 if (src_matches_dest_reg(&mov
->dest
.dest
, &mov
->src
[0].src
) &&
93 !mov
->src
[0].abs
&& !mov
->src
[0].negate
) {
94 for (unsigned i
= 0; i
< 4; i
++) {
95 if (mov
->src
[0].swizzle
[i
] == i
) {
96 mov
->dest
.write_mask
&= ~(1 << i
);
101 /* Only emit the instruction if it actually does something */
102 if (mov
->dest
.write_mask
) {
103 nir_instr_insert_before(&vec
->instr
, &mov
->instr
);
108 return channels_handled
;
112 has_replicated_dest(nir_alu_instr
*alu
)
114 return alu
->op
== nir_op_fdot_replicated2
||
115 alu
->op
== nir_op_fdot_replicated3
||
116 alu
->op
== nir_op_fdot_replicated4
||
117 alu
->op
== nir_op_fdph_replicated
;
120 /* Attempts to coalesce the "move" from the given source of the vec to the
121 * destination of the instruction generating the value. If, for whatever
122 * reason, we cannot coalesce the mmove, it does nothing and returns 0. We
123 * can then call insert_mov as normal.
126 try_coalesce(nir_alu_instr
*vec
, unsigned start_idx
, nir_shader
*shader
)
128 assert(start_idx
< nir_op_infos
[vec
->op
].num_inputs
);
130 /* We will only even try if the source is SSA */
131 if (!vec
->src
[start_idx
].src
.is_ssa
)
134 assert(vec
->src
[start_idx
].src
.ssa
);
136 /* If we are going to do a reswizzle, then the vecN operation must be the
137 * only use of the source value. We also can't have any source modifiers.
139 nir_foreach_use(vec
->src
[start_idx
].src
.ssa
, src
) {
140 if (src
->parent_instr
!= &vec
->instr
)
143 nir_alu_src
*alu_src
= exec_node_data(nir_alu_src
, src
, src
);
144 if (alu_src
->abs
|| alu_src
->negate
)
148 if (!list_empty(&vec
->src
[start_idx
].src
.ssa
->if_uses
))
151 if (vec
->src
[start_idx
].src
.ssa
->parent_instr
->type
!= nir_instr_type_alu
)
154 nir_alu_instr
*src_alu
=
155 nir_instr_as_alu(vec
->src
[start_idx
].src
.ssa
->parent_instr
);
157 if (has_replicated_dest(src_alu
)) {
158 /* The fdot instruction is special: It replicates its result to all
159 * components. This means that we can always rewrite its destination
160 * and we don't need to swizzle anything.
163 /* We only care about being able to re-swizzle the instruction if it is
164 * something that we can reswizzle. It must be per-component. The one
165 * exception to this is the fdotN instructions which implicitly splat
166 * their result out to all channels.
168 if (nir_op_infos
[src_alu
->op
].output_size
!= 0)
171 /* If we are going to reswizzle the instruction, we can't have any
172 * non-per-component sources either.
174 for (unsigned j
= 0; j
< nir_op_infos
[src_alu
->op
].num_inputs
; j
++)
175 if (nir_op_infos
[src_alu
->op
].input_sizes
[j
] != 0)
179 /* Stash off all of the ALU instruction's swizzles. */
180 uint8_t swizzles
[4][4];
181 for (unsigned j
= 0; j
< nir_op_infos
[src_alu
->op
].num_inputs
; j
++)
182 for (unsigned i
= 0; i
< 4; i
++)
183 swizzles
[j
][i
] = src_alu
->src
[j
].swizzle
[i
];
185 unsigned write_mask
= 0;
186 for (unsigned i
= start_idx
; i
< 4; i
++) {
187 if (!(vec
->dest
.write_mask
& (1 << i
)))
190 if (!vec
->src
[i
].src
.is_ssa
||
191 vec
->src
[i
].src
.ssa
!= &src_alu
->dest
.dest
.ssa
)
194 /* At this point, the give vec source matchese up with the ALU
195 * instruction so we can re-swizzle that component to match.
197 write_mask
|= 1 << i
;
198 if (has_replicated_dest(src_alu
)) {
199 /* Since the destination is a single replicated value, we don't need
200 * to do any reswizzling
203 for (unsigned j
= 0; j
< nir_op_infos
[src_alu
->op
].num_inputs
; j
++)
204 src_alu
->src
[j
].swizzle
[i
] = swizzles
[j
][vec
->src
[i
].swizzle
[0]];
207 /* Clear the no longer needed vec source */
208 nir_instr_rewrite_src(&vec
->instr
, &vec
->src
[i
].src
, NIR_SRC_INIT
);
211 nir_instr_rewrite_dest(&src_alu
->instr
, &src_alu
->dest
.dest
, vec
->dest
.dest
);
212 src_alu
->dest
.write_mask
= write_mask
;
218 lower_vec_to_movs_block(nir_block
*block
, void *void_state
)
220 struct vec_to_movs_state
*state
= void_state
;
221 nir_function_impl
*impl
= state
->impl
;
222 nir_shader
*shader
= impl
->function
->shader
;
224 nir_foreach_instr_safe(block
, instr
) {
225 if (instr
->type
!= nir_instr_type_alu
)
228 nir_alu_instr
*vec
= nir_instr_as_alu(instr
);
236 continue; /* The loop */
239 if (vec
->dest
.dest
.is_ssa
) {
240 /* Since we insert multiple MOVs, we have a register destination. */
241 nir_register
*reg
= nir_local_reg_create(impl
);
242 reg
->num_components
= vec
->dest
.dest
.ssa
.num_components
;
244 nir_ssa_def_rewrite_uses(&vec
->dest
.dest
.ssa
, nir_src_for_reg(reg
));
246 nir_instr_rewrite_dest(&vec
->instr
, &vec
->dest
.dest
,
247 nir_dest_for_reg(reg
));
250 unsigned finished_write_mask
= 0;
252 /* First, emit a MOV for all the src channels that are in the
253 * destination reg, in case other values we're populating in the dest
254 * might overwrite them.
256 for (unsigned i
= 0; i
< 4; i
++) {
257 if (!(vec
->dest
.write_mask
& (1 << i
)))
260 if (src_matches_dest_reg(&vec
->dest
.dest
, &vec
->src
[i
].src
)) {
261 finished_write_mask
|= insert_mov(vec
, i
, shader
);
266 /* Now, emit MOVs for all the other src channels. */
267 for (unsigned i
= 0; i
< 4; i
++) {
268 if (!(vec
->dest
.write_mask
& (1 << i
)))
271 if (!(finished_write_mask
& (1 << i
)))
272 finished_write_mask
|= try_coalesce(vec
, i
, shader
);
274 if (!(finished_write_mask
& (1 << i
)))
275 finished_write_mask
|= insert_mov(vec
, i
, shader
);
278 nir_instr_remove(&vec
->instr
);
280 state
->progress
= true;
287 nir_lower_vec_to_movs_impl(nir_function_impl
*impl
)
289 struct vec_to_movs_state state
= { impl
, false };
291 nir_foreach_block(impl
, lower_vec_to_movs_block
, &state
);
293 if (state
.progress
) {
294 nir_metadata_preserve(impl
, nir_metadata_block_index
|
295 nir_metadata_dominance
);
298 return state
.progress
;
302 nir_lower_vec_to_movs(nir_shader
*shader
)
304 bool progress
= false;
306 nir_foreach_function(shader
, function
) {
308 progress
= nir_lower_vec_to_movs_impl(function
->impl
) || progress
;