2 * Copyright © 2014 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Connor Abbott (cwabbott0@gmail.com)
25 * Jason Ekstrand (jason@jlekstrand.net)
30 * This lowering pass converts references to input/output variables with
31 * loads/stores to actual input/output intrinsics.
35 #include "nir_builder.h"
36 #include "nir_deref.h"
38 struct lower_io_state
{
41 int (*type_size
)(const struct glsl_type
*type
);
42 nir_variable_mode modes
;
43 nir_lower_io_options options
;
47 nir_assign_var_locations(struct exec_list
*var_list
, unsigned *size
,
48 int (*type_size
)(const struct glsl_type
*))
50 unsigned location
= 0;
52 nir_foreach_variable(var
, var_list
) {
54 * UBOs have their own address spaces, so don't count them towards the
55 * number of global uniforms
57 if ((var
->data
.mode
== nir_var_uniform
|| var
->data
.mode
== nir_var_shader_storage
) &&
58 var
->interface_type
!= NULL
)
61 var
->data
.driver_location
= location
;
62 location
+= type_size(var
->type
);
69 * Return true if the given variable is a per-vertex input/output array.
70 * (such as geometry shader inputs).
73 nir_is_per_vertex_io(const nir_variable
*var
, gl_shader_stage stage
)
75 if (var
->data
.patch
|| !glsl_type_is_array(var
->type
))
78 if (var
->data
.mode
== nir_var_shader_in
)
79 return stage
== MESA_SHADER_GEOMETRY
||
80 stage
== MESA_SHADER_TESS_CTRL
||
81 stage
== MESA_SHADER_TESS_EVAL
;
83 if (var
->data
.mode
== nir_var_shader_out
)
84 return stage
== MESA_SHADER_TESS_CTRL
;
90 get_io_offset(nir_builder
*b
, nir_deref_instr
*deref
,
91 nir_ssa_def
**vertex_index
,
92 int (*type_size
)(const struct glsl_type
*),
96 nir_deref_path_init(&path
, deref
, NULL
);
98 assert(path
.path
[0]->deref_type
== nir_deref_type_var
);
99 nir_deref_instr
**p
= &path
.path
[1];
101 /* For per-vertex input arrays (i.e. geometry shader inputs), keep the
102 * outermost array index separate. Process the rest normally.
104 if (vertex_index
!= NULL
) {
105 assert((*p
)->deref_type
== nir_deref_type_array
);
106 *vertex_index
= nir_ssa_for_src(b
, (*p
)->arr
.index
, 1);
110 if (path
.path
[0]->var
->data
.compact
) {
111 assert((*p
)->deref_type
== nir_deref_type_array
);
112 assert(glsl_type_is_scalar((*p
)->type
));
114 /* We always lower indirect dereferences for "compact" array vars. */
115 nir_const_value
*const_index
= nir_src_as_const_value((*p
)->arr
.index
);
118 const unsigned total_offset
= *component
+ const_index
->u32
[0];
119 const unsigned slot_offset
= total_offset
/ 4;
120 *component
= total_offset
% 4;
121 return nir_imm_int(b
, type_size(glsl_vec4_type()) * slot_offset
);
124 /* Just emit code and let constant-folding go to town */
125 nir_ssa_def
*offset
= nir_imm_int(b
, 0);
128 if ((*p
)->deref_type
== nir_deref_type_array
) {
129 unsigned size
= type_size((*p
)->type
);
132 nir_imul(b
, nir_imm_int(b
, size
),
133 nir_ssa_for_src(b
, (*p
)->arr
.index
, 1));
135 offset
= nir_iadd(b
, offset
, mul
);
136 } else if ((*p
)->deref_type
== nir_deref_type_struct
) {
137 /* p starts at path[1], so this is safe */
138 nir_deref_instr
*parent
= *(p
- 1);
140 unsigned field_offset
= 0;
141 for (unsigned i
= 0; i
< (*p
)->strct
.index
; i
++) {
142 field_offset
+= type_size(glsl_get_struct_field(parent
->type
, i
));
144 offset
= nir_iadd(b
, offset
, nir_imm_int(b
, field_offset
));
146 unreachable("Unsupported deref type");
150 nir_deref_path_finish(&path
);
155 static nir_intrinsic_instr
*
156 lower_load(nir_intrinsic_instr
*intrin
, struct lower_io_state
*state
,
157 nir_ssa_def
*vertex_index
, nir_variable
*var
, nir_ssa_def
*offset
,
160 const nir_shader
*nir
= state
->builder
.shader
;
161 nir_variable_mode mode
= var
->data
.mode
;
162 nir_ssa_def
*barycentric
= NULL
;
166 case nir_var_shader_in
:
167 if (nir
->info
.stage
== MESA_SHADER_FRAGMENT
&&
168 nir
->options
->use_interpolated_input_intrinsics
&&
169 var
->data
.interpolation
!= INTERP_MODE_FLAT
) {
170 assert(vertex_index
== NULL
);
172 nir_intrinsic_op bary_op
;
173 if (var
->data
.sample
||
174 (state
->options
& nir_lower_io_force_sample_interpolation
))
175 bary_op
= nir_intrinsic_load_barycentric_sample
;
176 else if (var
->data
.centroid
)
177 bary_op
= nir_intrinsic_load_barycentric_centroid
;
179 bary_op
= nir_intrinsic_load_barycentric_pixel
;
181 barycentric
= nir_load_barycentric(&state
->builder
, bary_op
,
182 var
->data
.interpolation
);
183 op
= nir_intrinsic_load_interpolated_input
;
185 op
= vertex_index
? nir_intrinsic_load_per_vertex_input
:
186 nir_intrinsic_load_input
;
189 case nir_var_shader_out
:
190 op
= vertex_index
? nir_intrinsic_load_per_vertex_output
:
191 nir_intrinsic_load_output
;
193 case nir_var_uniform
:
194 op
= nir_intrinsic_load_uniform
;
197 op
= nir_intrinsic_load_shared
;
200 unreachable("Unknown variable mode");
203 nir_intrinsic_instr
*load
=
204 nir_intrinsic_instr_create(state
->builder
.shader
, op
);
205 load
->num_components
= intrin
->num_components
;
207 nir_intrinsic_set_base(load
, var
->data
.driver_location
);
208 if (mode
== nir_var_shader_in
|| mode
== nir_var_shader_out
)
209 nir_intrinsic_set_component(load
, component
);
211 if (load
->intrinsic
== nir_intrinsic_load_uniform
)
212 nir_intrinsic_set_range(load
, state
->type_size(var
->type
));
215 load
->src
[0] = nir_src_for_ssa(vertex_index
);
216 load
->src
[1] = nir_src_for_ssa(offset
);
217 } else if (barycentric
) {
218 load
->src
[0] = nir_src_for_ssa(barycentric
);
219 load
->src
[1] = nir_src_for_ssa(offset
);
221 load
->src
[0] = nir_src_for_ssa(offset
);
227 static nir_intrinsic_instr
*
228 lower_store(nir_intrinsic_instr
*intrin
, struct lower_io_state
*state
,
229 nir_ssa_def
*vertex_index
, nir_variable
*var
, nir_ssa_def
*offset
,
232 nir_variable_mode mode
= var
->data
.mode
;
235 if (mode
== nir_var_shared
) {
236 op
= nir_intrinsic_store_shared
;
238 assert(mode
== nir_var_shader_out
);
239 op
= vertex_index
? nir_intrinsic_store_per_vertex_output
:
240 nir_intrinsic_store_output
;
243 nir_intrinsic_instr
*store
=
244 nir_intrinsic_instr_create(state
->builder
.shader
, op
);
245 store
->num_components
= intrin
->num_components
;
247 nir_src_copy(&store
->src
[0], &intrin
->src
[1], store
);
249 nir_intrinsic_set_base(store
, var
->data
.driver_location
);
251 if (mode
== nir_var_shader_out
)
252 nir_intrinsic_set_component(store
, component
);
254 nir_intrinsic_set_write_mask(store
, nir_intrinsic_write_mask(intrin
));
257 store
->src
[1] = nir_src_for_ssa(vertex_index
);
259 store
->src
[vertex_index
? 2 : 1] = nir_src_for_ssa(offset
);
264 static nir_intrinsic_instr
*
265 lower_atomic(nir_intrinsic_instr
*intrin
, struct lower_io_state
*state
,
266 nir_variable
*var
, nir_ssa_def
*offset
)
268 assert(var
->data
.mode
== nir_var_shared
);
271 switch (intrin
->intrinsic
) {
272 #define OP(O) case nir_intrinsic_deref_##O: op = nir_intrinsic_shared_##O; break;
286 OP(atomic_fcomp_swap
)
289 unreachable("Invalid atomic");
292 nir_intrinsic_instr
*atomic
=
293 nir_intrinsic_instr_create(state
->builder
.shader
, op
);
295 nir_intrinsic_set_base(atomic
, var
->data
.driver_location
);
297 atomic
->src
[0] = nir_src_for_ssa(offset
);
298 assert(nir_intrinsic_infos
[intrin
->intrinsic
].num_srcs
==
299 nir_intrinsic_infos
[op
].num_srcs
);
300 for (unsigned i
= 1; i
< nir_intrinsic_infos
[op
].num_srcs
; i
++) {
301 nir_src_copy(&atomic
->src
[i
], &intrin
->src
[i
], atomic
);
307 static nir_intrinsic_instr
*
308 lower_interpolate_at(nir_intrinsic_instr
*intrin
, struct lower_io_state
*state
,
309 nir_variable
*var
, nir_ssa_def
*offset
, unsigned component
)
311 assert(var
->data
.mode
== nir_var_shader_in
);
313 /* Ignore interpolateAt() for flat variables - flat is flat. */
314 if (var
->data
.interpolation
== INTERP_MODE_FLAT
)
315 return lower_load(intrin
, state
, NULL
, var
, offset
, component
);
317 nir_intrinsic_op bary_op
;
318 switch (intrin
->intrinsic
) {
319 case nir_intrinsic_interp_deref_at_centroid
:
320 bary_op
= (state
->options
& nir_lower_io_force_sample_interpolation
) ?
321 nir_intrinsic_load_barycentric_sample
:
322 nir_intrinsic_load_barycentric_centroid
;
324 case nir_intrinsic_interp_deref_at_sample
:
325 bary_op
= nir_intrinsic_load_barycentric_at_sample
;
327 case nir_intrinsic_interp_deref_at_offset
:
328 bary_op
= nir_intrinsic_load_barycentric_at_offset
;
331 unreachable("Bogus interpolateAt() intrinsic.");
334 nir_intrinsic_instr
*bary_setup
=
335 nir_intrinsic_instr_create(state
->builder
.shader
, bary_op
);
337 nir_ssa_dest_init(&bary_setup
->instr
, &bary_setup
->dest
, 2, 32, NULL
);
338 nir_intrinsic_set_interp_mode(bary_setup
, var
->data
.interpolation
);
340 if (intrin
->intrinsic
== nir_intrinsic_interp_deref_at_sample
||
341 intrin
->intrinsic
== nir_intrinsic_interp_deref_at_offset
)
342 nir_src_copy(&bary_setup
->src
[0], &intrin
->src
[1], bary_setup
);
344 nir_builder_instr_insert(&state
->builder
, &bary_setup
->instr
);
346 nir_intrinsic_instr
*load
=
347 nir_intrinsic_instr_create(state
->builder
.shader
,
348 nir_intrinsic_load_interpolated_input
);
349 load
->num_components
= intrin
->num_components
;
351 nir_intrinsic_set_base(load
, var
->data
.driver_location
);
352 nir_intrinsic_set_component(load
, component
);
354 load
->src
[0] = nir_src_for_ssa(&bary_setup
->dest
.ssa
);
355 load
->src
[1] = nir_src_for_ssa(offset
);
361 nir_lower_io_block(nir_block
*block
,
362 struct lower_io_state
*state
)
364 nir_builder
*b
= &state
->builder
;
365 const nir_shader_compiler_options
*options
= b
->shader
->options
;
366 bool progress
= false;
368 nir_foreach_instr_safe(instr
, block
) {
369 if (instr
->type
!= nir_instr_type_intrinsic
)
372 nir_intrinsic_instr
*intrin
= nir_instr_as_intrinsic(instr
);
374 switch (intrin
->intrinsic
) {
375 case nir_intrinsic_load_deref
:
376 case nir_intrinsic_store_deref
:
377 case nir_intrinsic_deref_atomic_add
:
378 case nir_intrinsic_deref_atomic_imin
:
379 case nir_intrinsic_deref_atomic_umin
:
380 case nir_intrinsic_deref_atomic_imax
:
381 case nir_intrinsic_deref_atomic_umax
:
382 case nir_intrinsic_deref_atomic_and
:
383 case nir_intrinsic_deref_atomic_or
:
384 case nir_intrinsic_deref_atomic_xor
:
385 case nir_intrinsic_deref_atomic_exchange
:
386 case nir_intrinsic_deref_atomic_comp_swap
:
387 case nir_intrinsic_deref_atomic_fadd
:
388 case nir_intrinsic_deref_atomic_fmin
:
389 case nir_intrinsic_deref_atomic_fmax
:
390 case nir_intrinsic_deref_atomic_fcomp_swap
:
391 /* We can lower the io for this nir instrinsic */
393 case nir_intrinsic_interp_deref_at_centroid
:
394 case nir_intrinsic_interp_deref_at_sample
:
395 case nir_intrinsic_interp_deref_at_offset
:
396 /* We can optionally lower these to load_interpolated_input */
397 if (options
->use_interpolated_input_intrinsics
)
400 /* We can't lower the io for this nir instrinsic, so skip it */
404 nir_deref_instr
*deref
= nir_src_as_deref(intrin
->src
[0]);
406 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
407 nir_variable_mode mode
= var
->data
.mode
;
409 if ((state
->modes
& mode
) == 0)
412 if (mode
!= nir_var_shader_in
&&
413 mode
!= nir_var_shader_out
&&
414 mode
!= nir_var_shared
&&
415 mode
!= nir_var_uniform
)
418 b
->cursor
= nir_before_instr(instr
);
420 const bool per_vertex
= nir_is_per_vertex_io(var
, b
->shader
->info
.stage
);
423 nir_ssa_def
*vertex_index
= NULL
;
424 unsigned component_offset
= var
->data
.location_frac
;
426 offset
= get_io_offset(b
, deref
, per_vertex
? &vertex_index
: NULL
,
427 state
->type_size
, &component_offset
);
429 nir_intrinsic_instr
*replacement
;
431 switch (intrin
->intrinsic
) {
432 case nir_intrinsic_load_deref
:
433 replacement
= lower_load(intrin
, state
, vertex_index
, var
, offset
,
437 case nir_intrinsic_store_deref
:
438 replacement
= lower_store(intrin
, state
, vertex_index
, var
, offset
,
442 case nir_intrinsic_deref_atomic_add
:
443 case nir_intrinsic_deref_atomic_imin
:
444 case nir_intrinsic_deref_atomic_umin
:
445 case nir_intrinsic_deref_atomic_imax
:
446 case nir_intrinsic_deref_atomic_umax
:
447 case nir_intrinsic_deref_atomic_and
:
448 case nir_intrinsic_deref_atomic_or
:
449 case nir_intrinsic_deref_atomic_xor
:
450 case nir_intrinsic_deref_atomic_exchange
:
451 case nir_intrinsic_deref_atomic_comp_swap
:
452 case nir_intrinsic_deref_atomic_fadd
:
453 case nir_intrinsic_deref_atomic_fmin
:
454 case nir_intrinsic_deref_atomic_fmax
:
455 case nir_intrinsic_deref_atomic_fcomp_swap
:
456 assert(vertex_index
== NULL
);
457 replacement
= lower_atomic(intrin
, state
, var
, offset
);
460 case nir_intrinsic_interp_deref_at_centroid
:
461 case nir_intrinsic_interp_deref_at_sample
:
462 case nir_intrinsic_interp_deref_at_offset
:
463 assert(vertex_index
== NULL
);
464 replacement
= lower_interpolate_at(intrin
, state
, var
, offset
,
472 if (nir_intrinsic_infos
[intrin
->intrinsic
].has_dest
) {
473 if (intrin
->dest
.is_ssa
) {
474 nir_ssa_dest_init(&replacement
->instr
, &replacement
->dest
,
475 intrin
->dest
.ssa
.num_components
,
476 intrin
->dest
.ssa
.bit_size
, NULL
);
477 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
,
478 nir_src_for_ssa(&replacement
->dest
.ssa
));
480 nir_dest_copy(&replacement
->dest
, &intrin
->dest
, &intrin
->instr
);
484 nir_instr_insert_before(&intrin
->instr
, &replacement
->instr
);
485 nir_instr_remove(&intrin
->instr
);
493 nir_lower_io_impl(nir_function_impl
*impl
,
494 nir_variable_mode modes
,
495 int (*type_size
)(const struct glsl_type
*),
496 nir_lower_io_options options
)
498 struct lower_io_state state
;
499 bool progress
= false;
501 nir_builder_init(&state
.builder
, impl
);
502 state
.dead_ctx
= ralloc_context(NULL
);
504 state
.type_size
= type_size
;
505 state
.options
= options
;
507 nir_foreach_block(block
, impl
) {
508 progress
|= nir_lower_io_block(block
, &state
);
511 ralloc_free(state
.dead_ctx
);
513 nir_metadata_preserve(impl
, nir_metadata_block_index
|
514 nir_metadata_dominance
);
519 nir_lower_io(nir_shader
*shader
, nir_variable_mode modes
,
520 int (*type_size
)(const struct glsl_type
*),
521 nir_lower_io_options options
)
523 bool progress
= false;
525 nir_foreach_function(function
, shader
) {
526 if (function
->impl
) {
527 progress
|= nir_lower_io_impl(function
->impl
, modes
,
536 * Return the offset source for a load/store intrinsic.
539 nir_get_io_offset_src(nir_intrinsic_instr
*instr
)
541 switch (instr
->intrinsic
) {
542 case nir_intrinsic_load_input
:
543 case nir_intrinsic_load_output
:
544 case nir_intrinsic_load_uniform
:
545 return &instr
->src
[0];
546 case nir_intrinsic_load_ubo
:
547 case nir_intrinsic_load_ssbo
:
548 case nir_intrinsic_load_per_vertex_input
:
549 case nir_intrinsic_load_per_vertex_output
:
550 case nir_intrinsic_load_interpolated_input
:
551 case nir_intrinsic_store_output
:
552 return &instr
->src
[1];
553 case nir_intrinsic_store_ssbo
:
554 case nir_intrinsic_store_per_vertex_output
:
555 return &instr
->src
[2];
562 * Return the vertex index source for a load/store per_vertex intrinsic.
565 nir_get_io_vertex_index_src(nir_intrinsic_instr
*instr
)
567 switch (instr
->intrinsic
) {
568 case nir_intrinsic_load_per_vertex_input
:
569 case nir_intrinsic_load_per_vertex_output
:
570 return &instr
->src
[0];
571 case nir_intrinsic_store_per_vertex_output
:
572 return &instr
->src
[1];