2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include "nir_builder.h"
27 #include "util/hash_table.h"
29 /* This file contains various little helpers for doing simple linking in
30 * NIR. Eventually, we'll probably want a full-blown varying packing
31 * implementation in here. Right now, it just deletes unused things.
35 * Returns the bits in the inputs_read, outputs_written, or
36 * system_values_read bitfield corresponding to this variable.
39 get_variable_io_mask(nir_variable
*var
, gl_shader_stage stage
)
41 if (var
->data
.location
< 0)
44 unsigned location
= var
->data
.patch
?
45 var
->data
.location
- VARYING_SLOT_PATCH0
: var
->data
.location
;
47 assert(var
->data
.mode
== nir_var_shader_in
||
48 var
->data
.mode
== nir_var_shader_out
||
49 var
->data
.mode
== nir_var_system_value
);
50 assert(var
->data
.location
>= 0);
52 const struct glsl_type
*type
= var
->type
;
53 if (nir_is_per_vertex_io(var
, stage
)) {
54 assert(glsl_type_is_array(type
));
55 type
= glsl_get_array_element(type
);
58 unsigned slots
= glsl_count_attribute_slots(type
, false);
59 return ((1ull << slots
) - 1) << location
;
63 tcs_add_output_reads(nir_shader
*shader
, uint64_t *read
, uint64_t *patches_read
)
65 nir_foreach_function(function
, shader
) {
69 nir_foreach_block(block
, function
->impl
) {
70 nir_foreach_instr(instr
, block
) {
71 if (instr
->type
!= nir_instr_type_intrinsic
)
74 nir_intrinsic_instr
*intrin
= nir_instr_as_intrinsic(instr
);
75 if (intrin
->intrinsic
!= nir_intrinsic_load_deref
)
79 nir_deref_instr_get_variable(nir_src_as_deref(intrin
->src
[0]));
81 if (var
->data
.mode
!= nir_var_shader_out
)
84 if (var
->data
.patch
) {
85 patches_read
[var
->data
.location_frac
] |=
86 get_variable_io_mask(var
, shader
->info
.stage
);
88 read
[var
->data
.location_frac
] |=
89 get_variable_io_mask(var
, shader
->info
.stage
);
97 * Helper for removing unused shader I/O variables, by demoting them to global
98 * variables (which may then by dead code eliminated).
102 * progress = nir_remove_unused_io_vars(producer,
103 * &producer->outputs,
104 * read, patches_read) ||
107 * The "used" should be an array of 4 uint64_ts (probably of VARYING_BIT_*)
108 * representing each .location_frac used. Note that for vector variables,
109 * only the first channel (.location_frac) is examined for deciding if the
113 nir_remove_unused_io_vars(nir_shader
*shader
, struct exec_list
*var_list
,
114 uint64_t *used_by_other_stage
,
115 uint64_t *used_by_other_stage_patches
)
117 bool progress
= false;
120 nir_foreach_variable_safe(var
, var_list
) {
122 used
= used_by_other_stage_patches
;
124 used
= used_by_other_stage
;
126 if (var
->data
.location
< VARYING_SLOT_VAR0
&& var
->data
.location
>= 0)
129 if (var
->data
.always_active_io
)
132 uint64_t other_stage
= used
[var
->data
.location_frac
];
134 if (!(other_stage
& get_variable_io_mask(var
, shader
->info
.stage
))) {
135 /* This one is invalid, make it a global variable instead */
136 var
->data
.location
= 0;
137 var
->data
.mode
= nir_var_global
;
139 exec_node_remove(&var
->node
);
140 exec_list_push_tail(&shader
->globals
, &var
->node
);
147 nir_fixup_deref_modes(shader
);
153 nir_remove_unused_varyings(nir_shader
*producer
, nir_shader
*consumer
)
155 assert(producer
->info
.stage
!= MESA_SHADER_FRAGMENT
);
156 assert(consumer
->info
.stage
!= MESA_SHADER_VERTEX
);
158 uint64_t read
[4] = { 0 }, written
[4] = { 0 };
159 uint64_t patches_read
[4] = { 0 }, patches_written
[4] = { 0 };
161 nir_foreach_variable(var
, &producer
->outputs
) {
162 if (var
->data
.patch
) {
163 patches_written
[var
->data
.location_frac
] |=
164 get_variable_io_mask(var
, producer
->info
.stage
);
166 written
[var
->data
.location_frac
] |=
167 get_variable_io_mask(var
, producer
->info
.stage
);
171 nir_foreach_variable(var
, &consumer
->inputs
) {
172 if (var
->data
.patch
) {
173 patches_read
[var
->data
.location_frac
] |=
174 get_variable_io_mask(var
, consumer
->info
.stage
);
176 read
[var
->data
.location_frac
] |=
177 get_variable_io_mask(var
, consumer
->info
.stage
);
181 /* Each TCS invocation can read data written by other TCS invocations,
182 * so even if the outputs are not used by the TES we must also make
183 * sure they are not read by the TCS before demoting them to globals.
185 if (producer
->info
.stage
== MESA_SHADER_TESS_CTRL
)
186 tcs_add_output_reads(producer
, read
, patches_read
);
188 bool progress
= false;
189 progress
= nir_remove_unused_io_vars(producer
, &producer
->outputs
, read
,
192 progress
= nir_remove_unused_io_vars(consumer
, &consumer
->inputs
, written
,
193 patches_written
) || progress
;
199 get_interp_type(nir_variable
*var
, const struct glsl_type
*type
,
200 bool default_to_smooth_interp
)
202 if (glsl_type_is_integer(type
))
203 return INTERP_MODE_FLAT
;
204 else if (var
->data
.interpolation
!= INTERP_MODE_NONE
)
205 return var
->data
.interpolation
;
206 else if (default_to_smooth_interp
)
207 return INTERP_MODE_SMOOTH
;
209 return INTERP_MODE_NONE
;
212 #define INTERPOLATE_LOC_SAMPLE 0
213 #define INTERPOLATE_LOC_CENTROID 1
214 #define INTERPOLATE_LOC_CENTER 2
217 get_interp_loc(nir_variable
*var
)
219 if (var
->data
.sample
)
220 return INTERPOLATE_LOC_SAMPLE
;
221 else if (var
->data
.centroid
)
222 return INTERPOLATE_LOC_CENTROID
;
224 return INTERPOLATE_LOC_CENTER
;
228 get_slot_component_masks_and_interp_types(struct exec_list
*var_list
,
230 uint8_t *interp_type
,
232 gl_shader_stage stage
,
233 bool default_to_smooth_interp
)
235 nir_foreach_variable_safe(var
, var_list
) {
236 assert(var
->data
.location
>= 0);
238 /* Only remap things that aren't built-ins.
239 * TODO: add TES patch support.
241 if (var
->data
.location
>= VARYING_SLOT_VAR0
&&
242 var
->data
.location
- VARYING_SLOT_VAR0
< 32) {
244 const struct glsl_type
*type
= var
->type
;
245 if (nir_is_per_vertex_io(var
, stage
)) {
246 assert(glsl_type_is_array(type
));
247 type
= glsl_get_array_element(type
);
250 unsigned location
= var
->data
.location
- VARYING_SLOT_VAR0
;
252 glsl_get_vector_elements(glsl_without_array(type
));
254 bool dual_slot
= glsl_type_is_dual_slot(glsl_without_array(type
));
255 unsigned slots
= glsl_count_attribute_slots(type
, false);
256 unsigned comps_slot2
= 0;
257 for (unsigned i
= 0; i
< slots
; i
++) {
258 interp_type
[location
+ i
] =
259 get_interp_type(var
, type
, default_to_smooth_interp
);
260 interp_loc
[location
+ i
] = get_interp_loc(var
);
264 comps
[location
+ i
] |= ((1 << comps_slot2
) - 1);
266 unsigned num_comps
= 4 - var
->data
.location_frac
;
267 comps_slot2
= (elements
* 2) - num_comps
;
269 /* Assume ARB_enhanced_layouts packing rules for doubles */
270 assert(var
->data
.location_frac
== 0 ||
271 var
->data
.location_frac
== 2);
272 assert(comps_slot2
<= 4);
274 comps
[location
+ i
] |=
275 ((1 << num_comps
) - 1) << var
->data
.location_frac
;
278 comps
[location
+ i
] |=
279 ((1 << elements
) - 1) << var
->data
.location_frac
;
293 remap_slots_and_components(struct exec_list
*var_list
, gl_shader_stage stage
,
294 struct varying_loc (*remap
)[4],
295 uint64_t *slots_used
, uint64_t *out_slots_read
)
297 uint64_t out_slots_read_tmp
= 0;
299 /* We don't touch builtins so just copy the bitmask */
300 uint64_t slots_used_tmp
=
301 *slots_used
& (((uint64_t)1 << (VARYING_SLOT_VAR0
- 1)) - 1);
303 nir_foreach_variable(var
, var_list
) {
304 assert(var
->data
.location
>= 0);
306 /* Only remap things that aren't built-ins */
307 if (var
->data
.location
>= VARYING_SLOT_VAR0
&&
308 var
->data
.location
- VARYING_SLOT_VAR0
< 32) {
309 assert(var
->data
.location
- VARYING_SLOT_VAR0
< 32);
311 const struct glsl_type
*type
= var
->type
;
312 if (nir_is_per_vertex_io(var
, stage
)) {
313 assert(glsl_type_is_array(type
));
314 type
= glsl_get_array_element(type
);
317 unsigned num_slots
= glsl_count_attribute_slots(type
, false);
318 bool used_across_stages
= false;
319 bool outputs_read
= false;
321 unsigned location
= var
->data
.location
- VARYING_SLOT_VAR0
;
322 struct varying_loc
*new_loc
= &remap
[location
][var
->data
.location_frac
];
324 uint64_t slots
= (((uint64_t)1 << num_slots
) - 1) << var
->data
.location
;
325 if (slots
& *slots_used
)
326 used_across_stages
= true;
328 if (slots
& *out_slots_read
)
331 if (new_loc
->location
) {
332 var
->data
.location
= new_loc
->location
;
333 var
->data
.location_frac
= new_loc
->component
;
336 if (var
->data
.always_active_io
) {
337 /* We can't apply link time optimisations (specifically array
338 * splitting) to these so we need to copy the existing mask
339 * otherwise we will mess up the mask for things like partially
342 if (used_across_stages
) {
344 *slots_used
& (((uint64_t)1 << num_slots
) - 1) << var
->data
.location
;
348 out_slots_read_tmp
|=
349 *out_slots_read
& (((uint64_t)1 << num_slots
) - 1) << var
->data
.location
;
353 for (unsigned i
= 0; i
< num_slots
; i
++) {
354 if (used_across_stages
)
355 slots_used_tmp
|= (uint64_t)1 << (var
->data
.location
+ i
);
358 out_slots_read_tmp
|= (uint64_t)1 << (var
->data
.location
+ i
);
364 *slots_used
= slots_used_tmp
;
365 *out_slots_read
= out_slots_read_tmp
;
368 /* If there are empty components in the slot compact the remaining components
369 * as close to component 0 as possible. This will make it easier to fill the
370 * empty components with components from a different slot in a following pass.
373 compact_components(nir_shader
*producer
, nir_shader
*consumer
, uint8_t *comps
,
374 uint8_t *interp_type
, uint8_t *interp_loc
,
375 bool default_to_smooth_interp
)
377 struct exec_list
*input_list
= &consumer
->inputs
;
378 struct exec_list
*output_list
= &producer
->outputs
;
379 struct varying_loc remap
[32][4] = {{{0}, {0}}};
381 /* Create a cursor for each interpolation type */
382 unsigned cursor
[4] = {0};
384 /* We only need to pass over one stage and we choose the consumer as it seems
385 * to cause a larger reduction in instruction counts (tested on i965).
387 nir_foreach_variable(var
, input_list
) {
389 /* Only remap things that aren't builtins.
390 * TODO: add TES patch support.
392 if (var
->data
.location
>= VARYING_SLOT_VAR0
&&
393 var
->data
.location
- VARYING_SLOT_VAR0
< 32) {
395 /* We can't repack xfb varyings. */
396 if (var
->data
.always_active_io
)
399 const struct glsl_type
*type
= var
->type
;
400 if (nir_is_per_vertex_io(var
, consumer
->info
.stage
)) {
401 assert(glsl_type_is_array(type
));
402 type
= glsl_get_array_element(type
);
405 /* Skip types that require more complex packing handling.
406 * TODO: add support for these types.
408 if (glsl_type_is_array(type
) ||
409 glsl_type_is_dual_slot(type
) ||
410 glsl_type_is_matrix(type
) ||
411 glsl_type_is_struct(type
) ||
412 glsl_type_is_64bit(type
))
415 /* We ignore complex types above and all other vector types should
416 * have been split into scalar variables by the lower_io_to_scalar
417 * pass. The only exception should by OpenGL xfb varyings.
419 if (glsl_get_vector_elements(type
) != 1)
422 unsigned location
= var
->data
.location
- VARYING_SLOT_VAR0
;
423 uint8_t used_comps
= comps
[location
];
425 /* If there are no empty components there is nothing more for us to do.
427 if (used_comps
== 0xf)
430 bool found_new_offset
= false;
431 uint8_t interp
= get_interp_type(var
, type
, default_to_smooth_interp
);
432 for (; cursor
[interp
] < 32; cursor
[interp
]++) {
433 uint8_t cursor_used_comps
= comps
[cursor
[interp
]];
435 /* We couldn't find anywhere to pack the varying continue on. */
436 if (cursor
[interp
] == location
&&
437 (var
->data
.location_frac
== 0 ||
438 cursor_used_comps
& ((1 << (var
->data
.location_frac
)) - 1)))
441 /* We can only pack varyings with matching interpolation types */
442 if (interp_type
[cursor
[interp
]] != interp
)
445 /* Interpolation loc must match also.
446 * TODO: i965 can handle these if they don't match, but the
447 * radeonsi nir backend handles everything as vec4s and so expects
448 * this to be the same for all components. We could make this
449 * check driver specfific or drop it if NIR ever become the only
452 if (interp_loc
[cursor
[interp
]] != get_interp_loc(var
))
455 /* If the slot is empty just skip it for now, compact_var_list()
456 * can be called after this function to remove empty slots for us.
457 * TODO: finish implementing compact_var_list() requires array and
460 if (!cursor_used_comps
)
463 uint8_t unused_comps
= ~cursor_used_comps
;
465 for (unsigned i
= 0; i
< 4; i
++) {
466 uint8_t new_var_comps
= 1 << i
;
467 if (unused_comps
& new_var_comps
) {
468 remap
[location
][var
->data
.location_frac
].component
= i
;
469 remap
[location
][var
->data
.location_frac
].location
=
470 cursor
[interp
] + VARYING_SLOT_VAR0
;
472 found_new_offset
= true;
474 /* Turn off the mask for the component we are remapping */
475 if (comps
[location
] & 1 << var
->data
.location_frac
) {
476 comps
[location
] ^= 1 << var
->data
.location_frac
;
477 comps
[cursor
[interp
]] |= new_var_comps
;
483 if (found_new_offset
)
490 remap_slots_and_components(input_list
, consumer
->info
.stage
, remap
,
491 &consumer
->info
.inputs_read
, &zero
);
492 remap_slots_and_components(output_list
, producer
->info
.stage
, remap
,
493 &producer
->info
.outputs_written
,
494 &producer
->info
.outputs_read
);
497 /* We assume that this has been called more-or-less directly after
498 * remove_unused_varyings. At this point, all of the varyings that we
499 * aren't going to be using have been completely removed and the
500 * inputs_read and outputs_written fields in nir_shader_info reflect
501 * this. Therefore, the total set of valid slots is the OR of the two
502 * sets of varyings; this accounts for varyings which one side may need
503 * to read/write even if the other doesn't. This can happen if, for
504 * instance, an array is used indirectly from one side causing it to be
505 * unsplittable but directly from the other.
508 nir_compact_varyings(nir_shader
*producer
, nir_shader
*consumer
,
509 bool default_to_smooth_interp
)
511 assert(producer
->info
.stage
!= MESA_SHADER_FRAGMENT
);
512 assert(consumer
->info
.stage
!= MESA_SHADER_VERTEX
);
514 uint8_t comps
[32] = {0};
515 uint8_t interp_type
[32] = {0};
516 uint8_t interp_loc
[32] = {0};
518 get_slot_component_masks_and_interp_types(&producer
->outputs
, comps
,
519 interp_type
, interp_loc
,
520 producer
->info
.stage
,
521 default_to_smooth_interp
);
522 get_slot_component_masks_and_interp_types(&consumer
->inputs
, comps
,
523 interp_type
, interp_loc
,
524 consumer
->info
.stage
,
525 default_to_smooth_interp
);
527 compact_components(producer
, consumer
, comps
, interp_type
, interp_loc
,
528 default_to_smooth_interp
);
532 * Mark XFB varyings as always_active_io in the consumer so the linking opts
536 nir_link_xfb_varyings(nir_shader
*producer
, nir_shader
*consumer
)
538 nir_variable
*input_vars
[MAX_VARYING
] = { 0 };
540 nir_foreach_variable(var
, &consumer
->inputs
) {
541 if (var
->data
.location
>= VARYING_SLOT_VAR0
&&
542 var
->data
.location
- VARYING_SLOT_VAR0
< MAX_VARYING
) {
544 unsigned location
= var
->data
.location
- VARYING_SLOT_VAR0
;
545 input_vars
[location
] = var
;
549 nir_foreach_variable(var
, &producer
->outputs
) {
550 if (var
->data
.location
>= VARYING_SLOT_VAR0
&&
551 var
->data
.location
- VARYING_SLOT_VAR0
< MAX_VARYING
) {
553 if (!var
->data
.always_active_io
)
556 unsigned location
= var
->data
.location
- VARYING_SLOT_VAR0
;
557 if (input_vars
[location
]) {
558 input_vars
[location
]->data
.always_active_io
= true;
565 try_replace_constant_input(nir_shader
*shader
,
566 nir_intrinsic_instr
*store_intr
)
568 nir_variable
*out_var
=
569 nir_deref_instr_get_variable(nir_src_as_deref(store_intr
->src
[0]));
571 if (out_var
->data
.mode
!= nir_var_shader_out
)
574 /* Skip types that require more complex handling.
575 * TODO: add support for these types.
577 if (glsl_type_is_array(out_var
->type
) ||
578 glsl_type_is_dual_slot(out_var
->type
) ||
579 glsl_type_is_matrix(out_var
->type
) ||
580 glsl_type_is_struct(out_var
->type
))
583 /* Limit this pass to scalars for now to keep things simple. Most varyings
584 * should have been lowered to scalars at this point anyway.
586 if (store_intr
->num_components
!= 1)
589 if (out_var
->data
.location
< VARYING_SLOT_VAR0
||
590 out_var
->data
.location
- VARYING_SLOT_VAR0
>= MAX_VARYING
)
593 nir_function_impl
*impl
= nir_shader_get_entrypoint(shader
);
596 nir_builder_init(&b
, impl
);
598 bool progress
= false;
599 nir_foreach_block(block
, impl
) {
600 nir_foreach_instr(instr
, block
) {
601 if (instr
->type
!= nir_instr_type_intrinsic
)
604 nir_intrinsic_instr
*intr
= nir_instr_as_intrinsic(instr
);
605 if (intr
->intrinsic
!= nir_intrinsic_load_deref
)
608 nir_variable
*in_var
=
609 nir_deref_instr_get_variable(nir_src_as_deref(intr
->src
[0]));
611 if (in_var
->data
.mode
!= nir_var_shader_in
)
614 if (in_var
->data
.location
!= out_var
->data
.location
||
615 in_var
->data
.location_frac
!= out_var
->data
.location_frac
)
618 b
.cursor
= nir_before_instr(instr
);
620 nir_load_const_instr
*out_const
=
621 nir_instr_as_load_const(store_intr
->src
[1].ssa
->parent_instr
);
623 /* Add new const to replace the input */
624 nir_ssa_def
*nconst
= nir_build_imm(&b
, store_intr
->num_components
,
625 intr
->dest
.ssa
.bit_size
,
628 nir_ssa_def_rewrite_uses(&intr
->dest
.ssa
, nir_src_for_ssa(nconst
));
638 nir_link_constant_varyings(nir_shader
*producer
, nir_shader
*consumer
)
640 /* TODO: Add support for more shader stage combinations */
641 if (consumer
->info
.stage
!= MESA_SHADER_FRAGMENT
||
642 (producer
->info
.stage
!= MESA_SHADER_VERTEX
&&
643 producer
->info
.stage
!= MESA_SHADER_TESS_EVAL
))
646 bool progress
= false;
648 nir_function_impl
*impl
= nir_shader_get_entrypoint(producer
);
650 /* If we find a store in the last block of the producer we can be sure this
651 * is the only possible value for this output.
653 nir_block
*last_block
= nir_impl_last_block(impl
);
654 nir_foreach_instr_reverse(instr
, last_block
) {
655 if (instr
->type
!= nir_instr_type_intrinsic
)
658 nir_intrinsic_instr
*intr
= nir_instr_as_intrinsic(instr
);
660 if (intr
->intrinsic
!= nir_intrinsic_store_deref
)
663 if (intr
->src
[1].ssa
->parent_instr
->type
!= nir_instr_type_load_const
) {
667 progress
|= try_replace_constant_input(consumer
, intr
);