2 * Copyright © 2014 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Connor Abbott (cwabbott0@gmail.com)
25 * Jason Ekstrand (jason@jlekstrand.net)
30 * This lowering pass converts references to input/output variables with
31 * loads/stores to actual input/output intrinsics.
35 #include "nir_builder.h"
37 struct lower_io_state
{
40 int (*type_size
)(const struct glsl_type
*type
);
41 nir_variable_mode modes
;
42 nir_lower_io_options options
;
46 nir_assign_var_locations(struct exec_list
*var_list
, unsigned *size
,
48 int (*type_size
)(const struct glsl_type
*))
50 unsigned location
= 0;
52 /* There are 32 regular and 32 patch varyings allowed */
54 for (unsigned i
= 0; i
< 64; i
++) {
55 for (unsigned j
= 0; j
< 2; j
++)
59 nir_foreach_variable(var
, var_list
) {
61 * UBO's have their own address spaces, so don't count them towards the
62 * number of global uniforms
64 if ((var
->data
.mode
== nir_var_uniform
|| var
->data
.mode
== nir_var_shader_storage
) &&
65 var
->interface_type
!= NULL
)
68 /* Make sure we give the same location to varyings packed with
69 * ARB_enhanced_layouts.
71 int idx
= var
->data
.location
- base_offset
;
72 if (base_offset
&& idx
>= 0) {
73 assert(idx
< ARRAY_SIZE(locations
));
75 if (locations
[idx
][var
->data
.index
] == -1) {
76 var
->data
.driver_location
= location
;
77 locations
[idx
][var
->data
.index
] = location
;
79 /* A dvec3 can be packed with a double we need special handling
80 * for this as we are packing across two locations.
82 if (glsl_get_base_type(var
->type
) == GLSL_TYPE_DOUBLE
&&
83 glsl_get_vector_elements(var
->type
) == 3) {
84 /* Hack around type_size functions that expect vectors to be
85 * padded out to vec4. If a float type is the same size as a
86 * double then the type size is padded to vec4, otherwise
87 * set the offset to two doubles which offsets the location
88 * past the first two components in dvec3 which were stored at
89 * the previous location.
91 unsigned dsize
= type_size(glsl_double_type());
93 dsize
== type_size(glsl_float_type()) ? dsize
: dsize
* 2;
95 locations
[idx
+ 1][var
->data
.index
] = location
+ offset
;
98 location
+= type_size(var
->type
);
100 var
->data
.driver_location
= locations
[idx
][var
->data
.index
];
103 var
->data
.driver_location
= location
;
104 location
+= type_size(var
->type
);
112 * Returns true if we're processing a stage whose inputs are arrays indexed
113 * by a vertex number (such as geometry shader inputs).
116 is_per_vertex_input(struct lower_io_state
*state
, nir_variable
*var
)
118 gl_shader_stage stage
= state
->builder
.shader
->stage
;
120 return var
->data
.mode
== nir_var_shader_in
&& !var
->data
.patch
&&
121 (stage
== MESA_SHADER_TESS_CTRL
||
122 stage
== MESA_SHADER_TESS_EVAL
||
123 stage
== MESA_SHADER_GEOMETRY
);
127 is_per_vertex_output(struct lower_io_state
*state
, nir_variable
*var
)
129 gl_shader_stage stage
= state
->builder
.shader
->stage
;
130 return var
->data
.mode
== nir_var_shader_out
&& !var
->data
.patch
&&
131 stage
== MESA_SHADER_TESS_CTRL
;
135 get_io_offset(nir_builder
*b
, nir_deref_var
*deref
,
136 nir_ssa_def
**vertex_index
,
137 int (*type_size
)(const struct glsl_type
*))
139 nir_deref
*tail
= &deref
->deref
;
141 /* For per-vertex input arrays (i.e. geometry shader inputs), keep the
142 * outermost array index separate. Process the rest normally.
144 if (vertex_index
!= NULL
) {
146 assert(tail
->deref_type
== nir_deref_type_array
);
147 nir_deref_array
*deref_array
= nir_deref_as_array(tail
);
149 nir_ssa_def
*vtx
= nir_imm_int(b
, deref_array
->base_offset
);
150 if (deref_array
->deref_array_type
== nir_deref_array_type_indirect
) {
151 vtx
= nir_iadd(b
, vtx
, nir_ssa_for_src(b
, deref_array
->indirect
, 1));
156 /* Just emit code and let constant-folding go to town */
157 nir_ssa_def
*offset
= nir_imm_int(b
, 0);
159 while (tail
->child
!= NULL
) {
160 const struct glsl_type
*parent_type
= tail
->type
;
163 if (tail
->deref_type
== nir_deref_type_array
) {
164 nir_deref_array
*deref_array
= nir_deref_as_array(tail
);
165 unsigned size
= type_size(tail
->type
);
167 offset
= nir_iadd(b
, offset
,
168 nir_imm_int(b
, size
* deref_array
->base_offset
));
170 if (deref_array
->deref_array_type
== nir_deref_array_type_indirect
) {
172 nir_imul(b
, nir_imm_int(b
, size
),
173 nir_ssa_for_src(b
, deref_array
->indirect
, 1));
175 offset
= nir_iadd(b
, offset
, mul
);
177 } else if (tail
->deref_type
== nir_deref_type_struct
) {
178 nir_deref_struct
*deref_struct
= nir_deref_as_struct(tail
);
180 unsigned field_offset
= 0;
181 for (unsigned i
= 0; i
< deref_struct
->index
; i
++) {
182 field_offset
+= type_size(glsl_get_struct_field(parent_type
, i
));
184 offset
= nir_iadd(b
, offset
, nir_imm_int(b
, field_offset
));
191 static nir_intrinsic_instr
*
192 lower_load(nir_intrinsic_instr
*intrin
, struct lower_io_state
*state
,
193 nir_ssa_def
*vertex_index
, nir_ssa_def
*offset
)
195 const nir_shader
*nir
= state
->builder
.shader
;
196 nir_variable
*var
= intrin
->variables
[0]->var
;
197 nir_variable_mode mode
= var
->data
.mode
;
198 nir_ssa_def
*barycentric
= NULL
;
202 case nir_var_shader_in
:
203 if (nir
->stage
== MESA_SHADER_FRAGMENT
&&
204 nir
->options
->use_interpolated_input_intrinsics
&&
205 var
->data
.interpolation
!= INTERP_MODE_FLAT
) {
206 assert(vertex_index
== NULL
);
208 nir_intrinsic_op bary_op
;
209 if (var
->data
.sample
||
210 (state
->options
& nir_lower_io_force_sample_interpolation
))
211 bary_op
= nir_intrinsic_load_barycentric_sample
;
212 else if (var
->data
.centroid
)
213 bary_op
= nir_intrinsic_load_barycentric_centroid
;
215 bary_op
= nir_intrinsic_load_barycentric_pixel
;
217 barycentric
= nir_load_barycentric(&state
->builder
, bary_op
,
218 var
->data
.interpolation
);
219 op
= nir_intrinsic_load_interpolated_input
;
221 op
= vertex_index
? nir_intrinsic_load_per_vertex_input
:
222 nir_intrinsic_load_input
;
225 case nir_var_shader_out
:
226 op
= vertex_index
? nir_intrinsic_load_per_vertex_output
:
227 nir_intrinsic_load_output
;
229 case nir_var_uniform
:
230 op
= nir_intrinsic_load_uniform
;
233 op
= nir_intrinsic_load_shared
;
236 unreachable("Unknown variable mode");
239 nir_intrinsic_instr
*load
= nir_intrinsic_instr_create(state
->mem_ctx
, op
);
240 load
->num_components
= intrin
->num_components
;
242 nir_intrinsic_set_base(load
, var
->data
.driver_location
);
243 if (mode
== nir_var_shader_in
|| mode
== nir_var_shader_out
)
244 nir_intrinsic_set_component(load
, var
->data
.location_frac
);
246 if (load
->intrinsic
== nir_intrinsic_load_uniform
)
247 nir_intrinsic_set_range(load
, state
->type_size(var
->type
));
250 load
->src
[0] = nir_src_for_ssa(vertex_index
);
251 load
->src
[1] = nir_src_for_ssa(offset
);
252 } else if (barycentric
) {
253 load
->src
[0] = nir_src_for_ssa(barycentric
);
254 load
->src
[1] = nir_src_for_ssa(offset
);
256 load
->src
[0] = nir_src_for_ssa(offset
);
262 static nir_intrinsic_instr
*
263 lower_store(nir_intrinsic_instr
*intrin
, struct lower_io_state
*state
,
264 nir_ssa_def
*vertex_index
, nir_ssa_def
*offset
)
266 nir_variable
*var
= intrin
->variables
[0]->var
;
267 nir_variable_mode mode
= var
->data
.mode
;
270 if (mode
== nir_var_shared
) {
271 op
= nir_intrinsic_store_shared
;
273 assert(mode
== nir_var_shader_out
);
274 op
= vertex_index
? nir_intrinsic_store_per_vertex_output
:
275 nir_intrinsic_store_output
;
278 nir_intrinsic_instr
*store
= nir_intrinsic_instr_create(state
->mem_ctx
, op
);
279 store
->num_components
= intrin
->num_components
;
281 nir_src_copy(&store
->src
[0], &intrin
->src
[0], store
);
283 nir_intrinsic_set_base(store
, var
->data
.driver_location
);
285 if (mode
== nir_var_shader_out
)
286 nir_intrinsic_set_component(store
, var
->data
.location_frac
);
288 nir_intrinsic_set_write_mask(store
, nir_intrinsic_write_mask(intrin
));
291 store
->src
[1] = nir_src_for_ssa(vertex_index
);
293 store
->src
[vertex_index
? 2 : 1] = nir_src_for_ssa(offset
);
298 static nir_intrinsic_instr
*
299 lower_atomic(nir_intrinsic_instr
*intrin
, struct lower_io_state
*state
,
302 nir_variable
*var
= intrin
->variables
[0]->var
;
304 assert(var
->data
.mode
== nir_var_shared
);
307 switch (intrin
->intrinsic
) {
308 #define OP(O) case nir_intrinsic_var_##O: op = nir_intrinsic_shared_##O; break;
321 unreachable("Invalid atomic");
324 nir_intrinsic_instr
*atomic
=
325 nir_intrinsic_instr_create(state
->mem_ctx
, op
);
327 nir_intrinsic_set_base(atomic
, var
->data
.driver_location
);
329 atomic
->src
[0] = nir_src_for_ssa(offset
);
330 for (unsigned i
= 0; i
< nir_op_infos
[intrin
->intrinsic
].num_inputs
; i
++) {
331 nir_src_copy(&atomic
->src
[i
+1], &intrin
->src
[i
], atomic
);
337 static nir_intrinsic_instr
*
338 lower_interpolate_at(nir_intrinsic_instr
*intrin
, struct lower_io_state
*state
,
341 nir_variable
*var
= intrin
->variables
[0]->var
;
343 assert(var
->data
.mode
== nir_var_shader_in
);
345 /* Ignore interpolateAt() for flat variables - flat is flat. */
346 if (var
->data
.interpolation
== INTERP_MODE_FLAT
)
347 return lower_load(intrin
, state
, NULL
, offset
);
349 nir_intrinsic_op bary_op
;
350 switch (intrin
->intrinsic
) {
351 case nir_intrinsic_interp_var_at_centroid
:
352 bary_op
= (state
->options
& nir_lower_io_force_sample_interpolation
) ?
353 nir_intrinsic_load_barycentric_sample
:
354 nir_intrinsic_load_barycentric_centroid
;
356 case nir_intrinsic_interp_var_at_sample
:
357 bary_op
= nir_intrinsic_load_barycentric_at_sample
;
359 case nir_intrinsic_interp_var_at_offset
:
360 bary_op
= nir_intrinsic_load_barycentric_at_offset
;
363 unreachable("Bogus interpolateAt() intrinsic.");
366 nir_intrinsic_instr
*bary_setup
=
367 nir_intrinsic_instr_create(state
->mem_ctx
, bary_op
);
369 nir_ssa_dest_init(&bary_setup
->instr
, &bary_setup
->dest
, 2, 32, NULL
);
370 nir_intrinsic_set_interp_mode(bary_setup
, var
->data
.interpolation
);
372 if (intrin
->intrinsic
!= nir_intrinsic_interp_var_at_centroid
)
373 nir_src_copy(&bary_setup
->src
[0], &intrin
->src
[0], bary_setup
);
375 nir_builder_instr_insert(&state
->builder
, &bary_setup
->instr
);
377 nir_intrinsic_instr
*load
=
378 nir_intrinsic_instr_create(state
->mem_ctx
,
379 nir_intrinsic_load_interpolated_input
);
380 load
->num_components
= intrin
->num_components
;
382 nir_intrinsic_set_base(load
, var
->data
.driver_location
);
383 nir_intrinsic_set_component(load
, var
->data
.location_frac
);
385 load
->src
[0] = nir_src_for_ssa(&bary_setup
->dest
.ssa
);
386 load
->src
[1] = nir_src_for_ssa(offset
);
392 nir_lower_io_block(nir_block
*block
,
393 struct lower_io_state
*state
)
395 nir_builder
*b
= &state
->builder
;
396 const nir_shader_compiler_options
*options
= b
->shader
->options
;
398 nir_foreach_instr_safe(instr
, block
) {
399 if (instr
->type
!= nir_instr_type_intrinsic
)
402 nir_intrinsic_instr
*intrin
= nir_instr_as_intrinsic(instr
);
404 switch (intrin
->intrinsic
) {
405 case nir_intrinsic_load_var
:
406 case nir_intrinsic_store_var
:
407 case nir_intrinsic_var_atomic_add
:
408 case nir_intrinsic_var_atomic_imin
:
409 case nir_intrinsic_var_atomic_umin
:
410 case nir_intrinsic_var_atomic_imax
:
411 case nir_intrinsic_var_atomic_umax
:
412 case nir_intrinsic_var_atomic_and
:
413 case nir_intrinsic_var_atomic_or
:
414 case nir_intrinsic_var_atomic_xor
:
415 case nir_intrinsic_var_atomic_exchange
:
416 case nir_intrinsic_var_atomic_comp_swap
:
417 /* We can lower the io for this nir instrinsic */
419 case nir_intrinsic_interp_var_at_centroid
:
420 case nir_intrinsic_interp_var_at_sample
:
421 case nir_intrinsic_interp_var_at_offset
:
422 /* We can optionally lower these to load_interpolated_input */
423 if (options
->use_interpolated_input_intrinsics
)
426 /* We can't lower the io for this nir instrinsic, so skip it */
430 nir_variable
*var
= intrin
->variables
[0]->var
;
431 nir_variable_mode mode
= var
->data
.mode
;
433 if ((state
->modes
& mode
) == 0)
436 if (mode
!= nir_var_shader_in
&&
437 mode
!= nir_var_shader_out
&&
438 mode
!= nir_var_shared
&&
439 mode
!= nir_var_uniform
)
442 b
->cursor
= nir_before_instr(instr
);
444 const bool per_vertex
=
445 is_per_vertex_input(state
, var
) || is_per_vertex_output(state
, var
);
448 nir_ssa_def
*vertex_index
= NULL
;
450 offset
= get_io_offset(b
, intrin
->variables
[0],
451 per_vertex
? &vertex_index
: NULL
,
454 nir_intrinsic_instr
*replacement
;
456 switch (intrin
->intrinsic
) {
457 case nir_intrinsic_load_var
:
458 replacement
= lower_load(intrin
, state
, vertex_index
, offset
);
461 case nir_intrinsic_store_var
:
462 replacement
= lower_store(intrin
, state
, vertex_index
, offset
);
465 case nir_intrinsic_var_atomic_add
:
466 case nir_intrinsic_var_atomic_imin
:
467 case nir_intrinsic_var_atomic_umin
:
468 case nir_intrinsic_var_atomic_imax
:
469 case nir_intrinsic_var_atomic_umax
:
470 case nir_intrinsic_var_atomic_and
:
471 case nir_intrinsic_var_atomic_or
:
472 case nir_intrinsic_var_atomic_xor
:
473 case nir_intrinsic_var_atomic_exchange
:
474 case nir_intrinsic_var_atomic_comp_swap
:
475 assert(vertex_index
== NULL
);
476 replacement
= lower_atomic(intrin
, state
, offset
);
479 case nir_intrinsic_interp_var_at_centroid
:
480 case nir_intrinsic_interp_var_at_sample
:
481 case nir_intrinsic_interp_var_at_offset
:
482 assert(vertex_index
== NULL
);
483 replacement
= lower_interpolate_at(intrin
, state
, offset
);
490 if (nir_intrinsic_infos
[intrin
->intrinsic
].has_dest
) {
491 if (intrin
->dest
.is_ssa
) {
492 nir_ssa_dest_init(&replacement
->instr
, &replacement
->dest
,
493 intrin
->dest
.ssa
.num_components
,
494 intrin
->dest
.ssa
.bit_size
, NULL
);
495 nir_ssa_def_rewrite_uses(&intrin
->dest
.ssa
,
496 nir_src_for_ssa(&replacement
->dest
.ssa
));
498 nir_dest_copy(&replacement
->dest
, &intrin
->dest
, state
->mem_ctx
);
502 nir_instr_insert_before(&intrin
->instr
, &replacement
->instr
);
503 nir_instr_remove(&intrin
->instr
);
510 nir_lower_io_impl(nir_function_impl
*impl
,
511 nir_variable_mode modes
,
512 int (*type_size
)(const struct glsl_type
*),
513 nir_lower_io_options options
)
515 struct lower_io_state state
;
517 nir_builder_init(&state
.builder
, impl
);
518 state
.mem_ctx
= ralloc_parent(impl
);
520 state
.type_size
= type_size
;
521 state
.options
= options
;
523 nir_foreach_block(block
, impl
) {
524 nir_lower_io_block(block
, &state
);
527 nir_metadata_preserve(impl
, nir_metadata_block_index
|
528 nir_metadata_dominance
);
532 nir_lower_io(nir_shader
*shader
, nir_variable_mode modes
,
533 int (*type_size
)(const struct glsl_type
*),
534 nir_lower_io_options options
)
536 nir_foreach_function(function
, shader
) {
537 if (function
->impl
) {
538 nir_lower_io_impl(function
->impl
, modes
, type_size
, options
);
544 * Return the offset soruce for a load/store intrinsic.
547 nir_get_io_offset_src(nir_intrinsic_instr
*instr
)
549 switch (instr
->intrinsic
) {
550 case nir_intrinsic_load_input
:
551 case nir_intrinsic_load_output
:
552 case nir_intrinsic_load_uniform
:
553 return &instr
->src
[0];
554 case nir_intrinsic_load_ubo
:
555 case nir_intrinsic_load_ssbo
:
556 case nir_intrinsic_load_per_vertex_input
:
557 case nir_intrinsic_load_per_vertex_output
:
558 case nir_intrinsic_load_interpolated_input
:
559 case nir_intrinsic_store_output
:
560 return &instr
->src
[1];
561 case nir_intrinsic_store_ssbo
:
562 case nir_intrinsic_store_per_vertex_output
:
563 return &instr
->src
[2];
570 * Return the vertex index source for a load/store per_vertex intrinsic.
573 nir_get_io_vertex_index_src(nir_intrinsic_instr
*instr
)
575 switch (instr
->intrinsic
) {
576 case nir_intrinsic_load_per_vertex_input
:
577 case nir_intrinsic_load_per_vertex_output
:
578 return &instr
->src
[0];
579 case nir_intrinsic_store_per_vertex_output
:
580 return &instr
->src
[1];