nir: Collect if shader uses cross-invocation or indirect I/O.
[mesa.git] / src / compiler / nir / nir_gather_info.c
1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "nir.h"
25 #include "nir_deref.h"
26 #include "main/menums.h"
27
28 static void
29 get_deref_info(nir_shader *shader, nir_variable *var, nir_deref_instr *deref,
30 bool *cross_invocation, bool *indirect)
31 {
32 *cross_invocation = false;
33 *indirect = false;
34
35 const bool per_vertex = nir_is_per_vertex_io(var, shader->info.stage);
36
37 nir_deref_path path;
38 nir_deref_path_init(&path, deref, NULL);
39 assert(path.path[0]->deref_type == nir_deref_type_var);
40 nir_deref_instr **p = &path.path[1];
41
42 /* Vertex index is the outermost array index. */
43 if (per_vertex) {
44 assert((*p)->deref_type == nir_deref_type_array);
45 nir_instr *vertex_index_instr = (*p)->arr.index.ssa->parent_instr;
46 *cross_invocation =
47 vertex_index_instr->type != nir_instr_type_intrinsic ||
48 nir_instr_as_intrinsic(vertex_index_instr)->intrinsic !=
49 nir_intrinsic_load_invocation_id;
50 p++;
51 }
52
53 /* We always lower indirect dereferences for "compact" array vars. */
54 if (!path.path[0]->var->data.compact) {
55 /* Non-compact array vars: find out if they are indirect. */
56 for (; *p; p++) {
57 if ((*p)->deref_type == nir_deref_type_array) {
58 *indirect |= !nir_src_is_const((*p)->arr.index);
59 } else if ((*p)->deref_type == nir_deref_type_struct) {
60 /* Struct indices are always constant. */
61 } else {
62 unreachable("Unsupported deref type");
63 }
64 }
65 }
66
67 nir_deref_path_finish(&path);
68 }
69
70 static void
71 set_io_mask(nir_shader *shader, nir_variable *var, int offset, int len,
72 nir_deref_instr *deref, bool is_output_read)
73 {
74 for (int i = 0; i < len; i++) {
75 assert(var->data.location != -1);
76
77 int idx = var->data.location + offset + i;
78 bool is_patch_generic = var->data.patch &&
79 idx != VARYING_SLOT_TESS_LEVEL_INNER &&
80 idx != VARYING_SLOT_TESS_LEVEL_OUTER &&
81 idx != VARYING_SLOT_BOUNDING_BOX0 &&
82 idx != VARYING_SLOT_BOUNDING_BOX1;
83 uint64_t bitfield;
84
85 if (is_patch_generic) {
86 assert(idx >= VARYING_SLOT_PATCH0 && idx < VARYING_SLOT_TESS_MAX);
87 bitfield = BITFIELD64_BIT(idx - VARYING_SLOT_PATCH0);
88 }
89 else {
90 assert(idx < VARYING_SLOT_MAX);
91 bitfield = BITFIELD64_BIT(idx);
92 }
93
94 bool cross_invocation;
95 bool indirect;
96 get_deref_info(shader, var, deref, &cross_invocation, &indirect);
97
98 if (var->data.mode == nir_var_shader_in) {
99 if (is_patch_generic) {
100 shader->info.patch_inputs_read |= bitfield;
101 if (indirect)
102 shader->info.patch_inputs_read_indirectly |= bitfield;
103 } else {
104 shader->info.inputs_read |= bitfield;
105 if (indirect)
106 shader->info.inputs_read_indirectly |= bitfield;
107 }
108
109 if (cross_invocation)
110 shader->info.tess.tcs_cross_invocation_inputs_read |= bitfield;
111
112 if (shader->info.stage == MESA_SHADER_FRAGMENT) {
113 shader->info.fs.uses_sample_qualifier |= var->data.sample;
114 }
115 } else {
116 assert(var->data.mode == nir_var_shader_out);
117 if (is_output_read) {
118 if (is_patch_generic) {
119 shader->info.patch_outputs_read |= bitfield;
120 if (indirect)
121 shader->info.patch_outputs_accessed_indirectly |= bitfield;
122 } else {
123 shader->info.outputs_read |= bitfield;
124 if (indirect)
125 shader->info.outputs_accessed_indirectly |= bitfield;
126 }
127
128 if (cross_invocation)
129 shader->info.tess.tcs_cross_invocation_outputs_read |= bitfield;
130 } else {
131 if (is_patch_generic) {
132 shader->info.patch_outputs_written |= bitfield;
133 if (indirect)
134 shader->info.patch_outputs_accessed_indirectly |= bitfield;
135 } else if (!var->data.read_only) {
136 shader->info.outputs_written |= bitfield;
137 if (indirect)
138 shader->info.outputs_accessed_indirectly |= bitfield;
139 }
140 }
141
142
143 if (var->data.fb_fetch_output)
144 shader->info.outputs_read |= bitfield;
145 }
146 }
147 }
148
149 /**
150 * Mark an entire variable as used. Caller must ensure that the variable
151 * represents a shader input or output.
152 */
153 static void
154 mark_whole_variable(nir_shader *shader, nir_variable *var,
155 nir_deref_instr *deref, bool is_output_read)
156 {
157 const struct glsl_type *type = var->type;
158
159 if (nir_is_per_vertex_io(var, shader->info.stage)) {
160 assert(glsl_type_is_array(type));
161 type = glsl_get_array_element(type);
162 }
163
164 const unsigned slots =
165 var->data.compact ? DIV_ROUND_UP(glsl_get_length(type), 4)
166 : glsl_count_attribute_slots(type, false);
167
168 set_io_mask(shader, var, 0, slots, deref, is_output_read);
169 }
170
171 static unsigned
172 get_io_offset(nir_deref_instr *deref, bool is_vertex_input, bool per_vertex)
173 {
174 unsigned offset = 0;
175
176 for (nir_deref_instr *d = deref; d; d = nir_deref_instr_parent(d)) {
177 if (d->deref_type == nir_deref_type_array) {
178 if (per_vertex && nir_deref_instr_parent(d)->deref_type == nir_deref_type_var)
179 break;
180
181 if (!nir_src_is_const(d->arr.index))
182 return -1;
183
184 offset += glsl_count_attribute_slots(d->type, is_vertex_input) *
185 nir_src_as_uint(d->arr.index);
186 }
187 /* TODO: we can get the offset for structs here see nir_lower_io() */
188 }
189
190 return offset;
191 }
192
193 /**
194 * Try to mark a portion of the given varying as used. Caller must ensure
195 * that the variable represents a shader input or output.
196 *
197 * If the index can't be interpreted as a constant, or some other problem
198 * occurs, then nothing will be marked and false will be returned.
199 */
200 static bool
201 try_mask_partial_io(nir_shader *shader, nir_variable *var,
202 nir_deref_instr *deref, bool is_output_read)
203 {
204 const struct glsl_type *type = var->type;
205 bool per_vertex = nir_is_per_vertex_io(var, shader->info.stage);
206
207 if (per_vertex) {
208 assert(glsl_type_is_array(type));
209 type = glsl_get_array_element(type);
210 }
211
212 /* The code below only handles:
213 *
214 * - Indexing into matrices
215 * - Indexing into arrays of (arrays, matrices, vectors, or scalars)
216 *
217 * For now, we just give up if we see varying structs and arrays of structs
218 * here marking the entire variable as used.
219 */
220 if (!(glsl_type_is_matrix(type) ||
221 (glsl_type_is_array(type) && !var->data.compact &&
222 (glsl_type_is_numeric(glsl_without_array(type)) ||
223 glsl_type_is_boolean(glsl_without_array(type)))))) {
224
225 /* If we don't know how to handle this case, give up and let the
226 * caller mark the whole variable as used.
227 */
228 return false;
229 }
230
231 unsigned offset = get_io_offset(deref, false, per_vertex);
232 if (offset == -1)
233 return false;
234
235 unsigned num_elems;
236 unsigned elem_width = 1;
237 unsigned mat_cols = 1;
238 if (glsl_type_is_array(type)) {
239 num_elems = glsl_get_aoa_size(type);
240 if (glsl_type_is_matrix(glsl_without_array(type)))
241 mat_cols = glsl_get_matrix_columns(glsl_without_array(type));
242 } else {
243 num_elems = glsl_get_matrix_columns(type);
244 }
245
246 /* double element width for double types that takes two slots */
247 if (glsl_type_is_dual_slot(glsl_without_array(type)))
248 elem_width *= 2;
249
250 if (offset >= num_elems * elem_width * mat_cols) {
251 /* Constant index outside the bounds of the matrix/array. This could
252 * arise as a result of constant folding of a legal GLSL program.
253 *
254 * Even though the spec says that indexing outside the bounds of a
255 * matrix/array results in undefined behaviour, we don't want to pass
256 * out-of-range values to set_io_mask() (since this could result in
257 * slots that don't exist being marked as used), so just let the caller
258 * mark the whole variable as used.
259 */
260 return false;
261 }
262
263 set_io_mask(shader, var, offset, elem_width, deref, is_output_read);
264 return true;
265 }
266
267 static void
268 gather_intrinsic_info(nir_intrinsic_instr *instr, nir_shader *shader,
269 void *dead_ctx)
270 {
271 switch (instr->intrinsic) {
272 case nir_intrinsic_demote:
273 case nir_intrinsic_demote_if:
274 shader->info.fs.uses_demote = true;
275 /* fallthrough: quads with helper lanes only might be discarded entirely */
276 case nir_intrinsic_discard:
277 case nir_intrinsic_discard_if:
278 assert(shader->info.stage == MESA_SHADER_FRAGMENT);
279 shader->info.fs.uses_discard = true;
280 break;
281
282 case nir_intrinsic_interp_deref_at_centroid:
283 case nir_intrinsic_interp_deref_at_sample:
284 case nir_intrinsic_interp_deref_at_offset:
285 case nir_intrinsic_interp_deref_at_vertex:
286 case nir_intrinsic_load_deref:
287 case nir_intrinsic_store_deref:{
288 nir_deref_instr *deref = nir_src_as_deref(instr->src[0]);
289 if (deref->mode == nir_var_shader_in ||
290 deref->mode == nir_var_shader_out) {
291 nir_variable *var = nir_deref_instr_get_variable(deref);
292 bool is_output_read = false;
293 if (var->data.mode == nir_var_shader_out &&
294 instr->intrinsic == nir_intrinsic_load_deref)
295 is_output_read = true;
296
297 if (!try_mask_partial_io(shader, var, deref, is_output_read))
298 mark_whole_variable(shader, var, deref, is_output_read);
299
300 /* We need to track which input_reads bits correspond to a
301 * dvec3/dvec4 input attribute */
302 if (shader->info.stage == MESA_SHADER_VERTEX &&
303 var->data.mode == nir_var_shader_in &&
304 glsl_type_is_dual_slot(glsl_without_array(var->type))) {
305 for (unsigned i = 0; i < glsl_count_attribute_slots(var->type, false); i++) {
306 int idx = var->data.location + i;
307 shader->info.vs.double_inputs |= BITFIELD64_BIT(idx);
308 }
309 }
310 }
311 break;
312 }
313
314 case nir_intrinsic_load_draw_id:
315 case nir_intrinsic_load_frag_coord:
316 case nir_intrinsic_load_point_coord:
317 case nir_intrinsic_load_front_face:
318 case nir_intrinsic_load_vertex_id:
319 case nir_intrinsic_load_vertex_id_zero_base:
320 case nir_intrinsic_load_base_vertex:
321 case nir_intrinsic_load_first_vertex:
322 case nir_intrinsic_load_is_indexed_draw:
323 case nir_intrinsic_load_base_instance:
324 case nir_intrinsic_load_instance_id:
325 case nir_intrinsic_load_sample_id:
326 case nir_intrinsic_load_sample_pos:
327 case nir_intrinsic_load_sample_mask_in:
328 case nir_intrinsic_load_primitive_id:
329 case nir_intrinsic_load_invocation_id:
330 case nir_intrinsic_load_local_invocation_id:
331 case nir_intrinsic_load_local_invocation_index:
332 case nir_intrinsic_load_work_group_id:
333 case nir_intrinsic_load_num_work_groups:
334 case nir_intrinsic_load_tess_coord:
335 case nir_intrinsic_load_tess_level_outer:
336 case nir_intrinsic_load_tess_level_inner:
337 case nir_intrinsic_load_patch_vertices_in:
338 shader->info.system_values_read |=
339 (1ull << nir_system_value_from_intrinsic(instr->intrinsic));
340 break;
341
342 case nir_intrinsic_quad_broadcast:
343 case nir_intrinsic_quad_swap_horizontal:
344 case nir_intrinsic_quad_swap_vertical:
345 case nir_intrinsic_quad_swap_diagonal:
346 if (shader->info.stage == MESA_SHADER_FRAGMENT)
347 shader->info.fs.needs_helper_invocations = true;
348 break;
349
350 case nir_intrinsic_end_primitive:
351 case nir_intrinsic_end_primitive_with_counter:
352 assert(shader->info.stage == MESA_SHADER_GEOMETRY);
353 shader->info.gs.uses_end_primitive = 1;
354 /* fall through */
355
356 case nir_intrinsic_emit_vertex:
357 case nir_intrinsic_emit_vertex_with_counter:
358 if (nir_intrinsic_stream_id(instr) > 0)
359 shader->info.gs.uses_streams = true;
360
361 break;
362
363 case nir_intrinsic_bindless_image_atomic_add:
364 case nir_intrinsic_bindless_image_atomic_and:
365 case nir_intrinsic_bindless_image_atomic_comp_swap:
366 case nir_intrinsic_bindless_image_atomic_dec_wrap:
367 case nir_intrinsic_bindless_image_atomic_exchange:
368 case nir_intrinsic_bindless_image_atomic_fadd:
369 case nir_intrinsic_bindless_image_atomic_imax:
370 case nir_intrinsic_bindless_image_atomic_imin:
371 case nir_intrinsic_bindless_image_atomic_inc_wrap:
372 case nir_intrinsic_bindless_image_atomic_or:
373 case nir_intrinsic_bindless_image_atomic_umax:
374 case nir_intrinsic_bindless_image_atomic_umin:
375 case nir_intrinsic_bindless_image_atomic_xor:
376 case nir_intrinsic_bindless_image_store:
377 case nir_intrinsic_bindless_image_store_raw_intel:
378 case nir_intrinsic_global_atomic_add:
379 case nir_intrinsic_global_atomic_and:
380 case nir_intrinsic_global_atomic_comp_swap:
381 case nir_intrinsic_global_atomic_exchange:
382 case nir_intrinsic_global_atomic_fadd:
383 case nir_intrinsic_global_atomic_fcomp_swap:
384 case nir_intrinsic_global_atomic_fmax:
385 case nir_intrinsic_global_atomic_fmin:
386 case nir_intrinsic_global_atomic_imax:
387 case nir_intrinsic_global_atomic_imin:
388 case nir_intrinsic_global_atomic_or:
389 case nir_intrinsic_global_atomic_umax:
390 case nir_intrinsic_global_atomic_umin:
391 case nir_intrinsic_global_atomic_xor:
392 case nir_intrinsic_image_atomic_add:
393 case nir_intrinsic_image_atomic_and:
394 case nir_intrinsic_image_atomic_comp_swap:
395 case nir_intrinsic_image_atomic_dec_wrap:
396 case nir_intrinsic_image_atomic_exchange:
397 case nir_intrinsic_image_atomic_fadd:
398 case nir_intrinsic_image_atomic_imax:
399 case nir_intrinsic_image_atomic_imin:
400 case nir_intrinsic_image_atomic_inc_wrap:
401 case nir_intrinsic_image_atomic_or:
402 case nir_intrinsic_image_atomic_umax:
403 case nir_intrinsic_image_atomic_umin:
404 case nir_intrinsic_image_atomic_xor:
405 case nir_intrinsic_image_deref_atomic_add:
406 case nir_intrinsic_image_deref_atomic_and:
407 case nir_intrinsic_image_deref_atomic_comp_swap:
408 case nir_intrinsic_image_deref_atomic_dec_wrap:
409 case nir_intrinsic_image_deref_atomic_exchange:
410 case nir_intrinsic_image_deref_atomic_fadd:
411 case nir_intrinsic_image_deref_atomic_imax:
412 case nir_intrinsic_image_deref_atomic_imin:
413 case nir_intrinsic_image_deref_atomic_inc_wrap:
414 case nir_intrinsic_image_deref_atomic_or:
415 case nir_intrinsic_image_deref_atomic_umax:
416 case nir_intrinsic_image_deref_atomic_umin:
417 case nir_intrinsic_image_deref_atomic_xor:
418 case nir_intrinsic_image_deref_store:
419 case nir_intrinsic_image_deref_store_raw_intel:
420 case nir_intrinsic_image_store:
421 case nir_intrinsic_image_store_raw_intel:
422 case nir_intrinsic_ssbo_atomic_add:
423 case nir_intrinsic_ssbo_atomic_add_ir3:
424 case nir_intrinsic_ssbo_atomic_and:
425 case nir_intrinsic_ssbo_atomic_and_ir3:
426 case nir_intrinsic_ssbo_atomic_comp_swap:
427 case nir_intrinsic_ssbo_atomic_comp_swap_ir3:
428 case nir_intrinsic_ssbo_atomic_exchange:
429 case nir_intrinsic_ssbo_atomic_exchange_ir3:
430 case nir_intrinsic_ssbo_atomic_fadd:
431 case nir_intrinsic_ssbo_atomic_fcomp_swap:
432 case nir_intrinsic_ssbo_atomic_fmax:
433 case nir_intrinsic_ssbo_atomic_fmin:
434 case nir_intrinsic_ssbo_atomic_imax:
435 case nir_intrinsic_ssbo_atomic_imax_ir3:
436 case nir_intrinsic_ssbo_atomic_imin:
437 case nir_intrinsic_ssbo_atomic_imin_ir3:
438 case nir_intrinsic_ssbo_atomic_or:
439 case nir_intrinsic_ssbo_atomic_or_ir3:
440 case nir_intrinsic_ssbo_atomic_umax:
441 case nir_intrinsic_ssbo_atomic_umax_ir3:
442 case nir_intrinsic_ssbo_atomic_umin:
443 case nir_intrinsic_ssbo_atomic_umin_ir3:
444 case nir_intrinsic_ssbo_atomic_xor:
445 case nir_intrinsic_ssbo_atomic_xor_ir3:
446 case nir_intrinsic_store_global:
447 case nir_intrinsic_store_global_ir3:
448 case nir_intrinsic_store_ssbo:
449 case nir_intrinsic_store_ssbo_ir3:
450 /* Only set this for globally visible memory, not scratch and not
451 * shared.
452 */
453 shader->info.writes_memory = true;
454 break;
455
456 default:
457 break;
458 }
459 }
460
461 static void
462 gather_tex_info(nir_tex_instr *instr, nir_shader *shader)
463 {
464 if (shader->info.stage == MESA_SHADER_FRAGMENT &&
465 nir_tex_instr_has_implicit_derivative(instr))
466 shader->info.fs.needs_helper_invocations = true;
467
468 switch (instr->op) {
469 case nir_texop_tg4:
470 shader->info.uses_texture_gather = true;
471 break;
472 default:
473 break;
474 }
475 }
476
477 static void
478 gather_alu_info(nir_alu_instr *instr, nir_shader *shader)
479 {
480 switch (instr->op) {
481 case nir_op_fddx:
482 case nir_op_fddy:
483 shader->info.uses_fddx_fddy = true;
484 /* Fall through */
485 case nir_op_fddx_fine:
486 case nir_op_fddy_fine:
487 case nir_op_fddx_coarse:
488 case nir_op_fddy_coarse:
489 if (shader->info.stage == MESA_SHADER_FRAGMENT)
490 shader->info.fs.needs_helper_invocations = true;
491 break;
492 default:
493 break;
494 }
495
496 shader->info.uses_64bit |= instr->dest.dest.ssa.bit_size == 64;
497 unsigned num_srcs = nir_op_infos[instr->op].num_inputs;
498 for (unsigned i = 0; i < num_srcs; i++) {
499 shader->info.uses_64bit |= nir_src_bit_size(instr->src[i].src) == 64;
500 }
501 }
502
503 static void
504 gather_info_block(nir_block *block, nir_shader *shader, void *dead_ctx)
505 {
506 nir_foreach_instr(instr, block) {
507 switch (instr->type) {
508 case nir_instr_type_alu:
509 gather_alu_info(nir_instr_as_alu(instr), shader);
510 break;
511 case nir_instr_type_intrinsic:
512 gather_intrinsic_info(nir_instr_as_intrinsic(instr), shader, dead_ctx);
513 break;
514 case nir_instr_type_tex:
515 gather_tex_info(nir_instr_as_tex(instr), shader);
516 break;
517 case nir_instr_type_call:
518 assert(!"nir_shader_gather_info only works if functions are inlined");
519 break;
520 default:
521 break;
522 }
523 }
524 }
525
526 void
527 nir_shader_gather_info(nir_shader *shader, nir_function_impl *entrypoint)
528 {
529 shader->info.num_textures = 0;
530 shader->info.num_images = 0;
531 shader->info.last_msaa_image = -1;
532
533 nir_foreach_variable(var, &shader->uniforms) {
534 /* Bindless textures and images don't use non-bindless slots. */
535 if (var->data.bindless)
536 continue;
537
538 shader->info.num_textures += glsl_type_get_sampler_count(var->type);
539 shader->info.num_images += glsl_type_get_image_count(var->type);
540
541 /* Assuming image slots don't have holes (e.g. OpenGL) */
542 if (glsl_type_is_image(var->type) &&
543 glsl_get_sampler_dim(var->type) == GLSL_SAMPLER_DIM_MS)
544 shader->info.last_msaa_image = shader->info.num_images - 1;
545 }
546
547 shader->info.inputs_read = 0;
548 shader->info.outputs_written = 0;
549 shader->info.outputs_read = 0;
550 shader->info.patch_outputs_read = 0;
551 shader->info.patch_inputs_read = 0;
552 shader->info.patch_outputs_written = 0;
553 shader->info.system_values_read = 0;
554 shader->info.inputs_read_indirectly = 0;
555 shader->info.outputs_accessed_indirectly = 0;
556 shader->info.patch_inputs_read_indirectly = 0;
557 shader->info.patch_outputs_accessed_indirectly = 0;
558
559 if (shader->info.stage == MESA_SHADER_VERTEX) {
560 shader->info.vs.double_inputs = 0;
561 }
562 if (shader->info.stage == MESA_SHADER_FRAGMENT) {
563 shader->info.fs.uses_sample_qualifier = false;
564 shader->info.fs.uses_discard = false;
565 shader->info.fs.uses_demote = false;
566 shader->info.fs.needs_helper_invocations = false;
567 }
568 if (shader->info.stage == MESA_SHADER_TESS_CTRL) {
569 shader->info.tess.tcs_cross_invocation_inputs_read = 0;
570 shader->info.tess.tcs_cross_invocation_outputs_read = 0;
571 }
572
573 shader->info.writes_memory = shader->info.has_transform_feedback_varyings;
574
575 void *dead_ctx = ralloc_context(NULL);
576 nir_foreach_block(block, entrypoint) {
577 gather_info_block(block, shader, dead_ctx);
578 }
579 ralloc_free(dead_ctx);
580 }